Frage zu HTTP-Header-Antwort



  • Hallo,
    ich schreibe zur Zeit mal wieder an meinem HTTP-client in C++ rum. Heute habe ich das Problem bekommen, dass bei dem Antwort-Paket, welches ich von meiner Seite angefordert habe, weder die 'chunked transfer'-Option, noch die 'content-length' angegeben wurde, weshalb mein Programm nun versagt. Meine Frage ist, ob dies nur ein Server-Fehler (Cloudflare) ist, oder ob ich eine Möglichkeit übersehen habe, wie man dennoch solche Pakete Pharsen kann.

    Hier folgt nun der Code; ich möchte noch erwähnen, dass der "Fehler" in Zeile 281 auftritt. Und ja, ich habe den ganzen Code drangehängt, damit man den Fehler selbst sehen kann. 🙄

    #include <iostream>
    #include <fstream>
    #include <cstdlib>
    #include <sys/socket.h> // socket(), connect()
    #include <arpa/inet.h> // sockaddr_in
    #include <netdb.h> // gethostbyname(), hostent
    #include <unistd.h>
    
    #define DEBUG_FLAG true
    
    using namespace std;
    
    class HTTPRequest {
    public:
        string RequestHost;
        string RequestPath;
        string RequestDataType;
        string RequestData;
        string Response;
        string ResponseProtocol;
        int ResponseCode;
        string ResponseCodeMessage;
        string RequestMessage;
        string ResponseDate;
        string ResponseVia;
        string ResponseDataType;
    private:
        bool Initialized = false;
        hostent* RequestHostResolved;
        int ResponseType = 0; // 0 = Unknown Size , 1 = Normal , 2 = Chunked
    
        string getLine(int socket) {
            string line;
            while (true) {
                char c;
                if (read(socket, &c, 1) <= 0) {
                    this->RequestMessage = "Error on receiving";
                    cerr << this->RequestMessage << endl;
                    return "";
                }
                if (c == '\n') {
                    return line;
                }
                line += c;
            }
        }
    
        void replaceAll(string* str, string from, string to) {
            string temp;
            for (int i = 0; i < str->length(); i++) {
                if (str->c_str()[i] != ' ') {
                    temp += str->c_str()[i];
                } else {
                    temp += '%';
                    temp += '2';
                    temp += '0';
                }
            }
            *str = temp;
        }
    public:
    
        HTTPRequest() {
            this->setHost("example.com");
            this->setPath("/");
            this->setDataType("Undefined");
            this->setData("");
            this->Response = "";
            this->ResponseProtocol = "Unknown";
            this->ResponseCode = 000;
            this->ResponseCodeMessage = "";
            this->RequestMessage = "";
            this->ResponseVia = "Not defined";
            this->ResponseDataType = "Not defined";
    
            this->Initialized = true;
        }
    
        void setPath(string GivenPath) {
            this->RequestPath = GivenPath;
        }
    
        void setData(string GivenData) {
            this->RequestData = GivenData;
        }
    
        void setData(string GivenDataType, string GivenData) {
            this->RequestData = GivenData;
            this->RequestDataType = GivenDataType;
        }
    
        void setDataType(string GivenDataType) {
            this->RequestDataType = GivenDataType;
        }
    
        bool setHost(string GivenHost) {
            this->Initialized = true;
            this->RequestHost = GivenHost;
            cout << "Resolving '" << this->RequestHost << "'" << endl;
    
            this->RequestHostResolved = gethostbyname(this->RequestHost.c_str());
    
            if (RequestHostResolved == NULL) {
                this->RequestMessage = "Can't resolve hostname!";
                cerr << this->RequestMessage << endl;
                return false;
            }
    
            if (RequestHostResolved->h_addrtype != AF_INET) {
                this->RequestMessage = "ERROR. Only IPv4 is supported (Unknown type)!";
                cerr << this->RequestMessage << endl;
                return false;
            }
    
            if (RequestHostResolved->h_length != 4) {
                this->RequestMessage = "ERROR. Only IPv4 is supported (Wrong type)!";
                cerr << this->RequestMessage << endl;
                return false;
            }
            if (DEBUG_FLAG) {
                cout << this->RequestHost << " resolved." << endl;
            }
        }
    
        bool perform() {
            if (!Initialized) {
                this->RequestMessage = "Error. The request is not fully initialized!";
                cerr << this->RequestMessage << endl;
                return false;
            }
            this->Response = "";
            cout << "Connecting to '" << this->RequestHost << "'" << endl;
    
            //Ask for a socket to communicate
            int Socket = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
            if (Socket == -1) {
                this->RequestMessage = "Can't get a free socket!";
                cerr << this->RequestMessage << endl;
                return false;
            }
    
            //Bind adresses and go!
            sockaddr_in service_addr;
            service_addr.sin_family = AF_INET;
            service_addr.sin_addr.s_addr = inet_addr(inet_ntoa(*(struct in_addr*) (RequestHostResolved->h_addr_list[0])));
            service_addr.sin_port = htons(80); //Use default HTTPport
    
            if (connect(Socket, (struct sockaddr *) &service_addr, sizeof (service_addr)) < 0) {
                this->RequestMessage = "Connection failed!";
                cerr << this->RequestMessage << endl;
                return false;
            } else {
                if (DEBUG_FLAG) {
                    cout << "Successfully connected." << endl;
                }
            }
    
            replaceAll(&this->RequestPath, " ", "%20");
            replaceAll(&this->RequestData, " ", "%20");
    
            //Generate the HTTP-package
            string HTTPrequest = "\n";
            if (this->RequestDataType == "POST") {
                HTTPrequest.clear();
                HTTPrequest.append("POST ");
                HTTPrequest.append(this->RequestPath.c_str());
                HTTPrequest.append(" HTTP/1.1\r\nHost: ");
                HTTPrequest.append(this->RequestHost.c_str());
                HTTPrequest.append("\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: ");
                HTTPrequest.append(to_string(this->RequestData.length()));
                HTTPrequest.append("\r\nConnection: close\r\n\r\n");
                HTTPrequest.append(this->RequestData.c_str());
                HTTPrequest.append("\r\n");
                //HTTPrequest = "POST /response.php HTTP/1.1\r\nHost: example.com\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 23\r\nConnection: close\r\n\r\nuser=simon&project=nope\r\n";
            }
            if (this->RequestDataType == "GET") {
                HTTPrequest.clear();
                HTTPrequest.append("GET ");
                HTTPrequest.append(this->RequestPath.c_str());
                HTTPrequest.append("?");
                HTTPrequest.append(this->RequestData);
                HTTPrequest.append(" HTTP/1.1\r\nHost: ");
                HTTPrequest.append(this->RequestHost);
                HTTPrequest.append("\r\nConnection: close\r\n\r\n");
                //HTTPrequest = "GET /response.php?user=simon&project=nope HTTP/1.1\r\nHost: example.com\r\nConnection: close\r\n\r\n";
            }
    
            if (this->RequestDataType == "" || this->RequestDataType == "Undefined") {
                HTTPrequest.clear();
                HTTPrequest.append("GET ");
                HTTPrequest.append(this->RequestPath.c_str());
                HTTPrequest.append(" HTTP/1.1\r\nHost: ");
                HTTPrequest.append(this->RequestHost);
                HTTPrequest.append("\r\nConnection: close\r\n\r\n");
                //HTTPrequest = "GET /response.php HTTP/1.1\r\nHost: example.com\r\nConnection: close\r\n\r\n";
            }
    
            if (DEBUG_FLAG) {
                cout << "\nHTTP-REQUEST\n" << HTTPrequest << "HTTP-REQUEST END\n" << endl;
            }
    
            if (write(Socket, HTTPrequest.c_str(), HTTPrequest.size()) != HTTPrequest.size()) {
                this->RequestMessage = "Connection error. Did the connection abort?";
                cerr << this->RequestMessage << endl;
                return false;
            }
    
            //Pharse the header package for information
            this->ResponseCode = 100;
            string HTTPHeader;
            while (ResponseCode == 100) {
                HTTPHeader += getLine(Socket);
                this->ResponseProtocol = HTTPHeader.substr(0, HTTPHeader.find(' '));
                this->ResponseCode = atoi(HTTPHeader.substr(HTTPHeader.find(' ') + 1, 3).c_str());
                if (this->ResponseCode == 100) {
                    HTTPHeader += getLine(Socket); //Ignore the empty line after 'Continue' (Code 100)
                }
            }
    
            if (DEBUG_FLAG) {
                cout << "Protocol used: " << this->ResponseProtocol << endl;
                cout << "Response code: " << this->ResponseCode << endl;
            }
    
            if (this->ResponseCode != 200) {
                this->ResponseCodeMessage = HTTPHeader;
                return false;
            }
    
            int size = -1;
    
            if (DEBUG_FLAG) {
                cout << "\nHTTP-HEAD-ANSWER\n";
            }
    
            this->ResponseType = -1;
            while (true) {
                HTTPHeader = getLine(Socket);
                if (DEBUG_FLAG) {
                    cout << HTTPHeader << endl;
                }
                if (HTTPHeader == "\r") //Header ended (One clean line)
                {
                    break;
                }
                string option = HTTPHeader.substr(0, HTTPHeader.find(':') + 1); //The option to set
                if (option == "Content-Length:") {
                    size = atoi(HTTPHeader.substr(HTTPHeader.find(':') + 2, (HTTPHeader.length() - HTTPHeader.find(':') + 2)).c_str());
                    if (size == -1) {
                        this->ResponseType = 0;
                    } else {
                        this->ResponseType = 1;
                    }
                }
                if (option == "Transfer-Encoding:") {
                    string transferEncoding;
                    transferEncoding = HTTPHeader.substr(HTTPHeader.find(':') + 2, (HTTPHeader.length() - HTTPHeader.find(':') + 2));
                    if (transferEncoding == "chunked\r") {
                        this->ResponseType = 2;
                    }
                }
                if (option == "Date:") {
                    this->ResponseDate = HTTPHeader.substr(HTTPHeader.find(':') + 2, (HTTPHeader.length() - HTTPHeader.find(':') + 2));
                }
                if (option == "Via:") {
                    this->ResponseVia = HTTPHeader.substr(HTTPHeader.find(':') + 2, (HTTPHeader.length() - HTTPHeader.find(':') + 2));
                }
                if (option == "Content-Type:") {
                    this->ResponseDataType = HTTPHeader.substr(HTTPHeader.find(':') + 2, (HTTPHeader.length() - HTTPHeader.find(':') + 2));
                }
            }
    
            if (DEBUG_FLAG) {
                cout << "HTTP-HEAD-ANSWER END" << endl;
            }
    
            //Now get the content
            cout << "Downloading ";
    
            //If no size or other options defined
            if(this->ResponseType == -1) {
                this->RequestMessage = "Can't download the requested content. There was no size given and no 'chunked' option set!";
                        cerr << this->RequestMessage << endl;
                        return false;
            }
            //If not size is defined
            if (this->ResponseType == 0) {
                int receivedBytes = -1;
                cout << "(Unknown filesize)..." << endl;
                char buffer = '\0';
                while (receivedBytes != 0) //If read() return we reached end of file
                {
                    if ((receivedBytes = read(Socket, &buffer, 1)) < 0) {
                        this->RequestMessage = "Connection error. Did the connection abort?";
                        cerr << this->RequestMessage << endl;
                        return false;
                    }
                    this->Response += buffer;
                    if(this->Response.length() % 1024 == 0) {
                        cout << "."; //One point for each KiB
                    }
                }
                cout << endl << this->Response.length() << " bytes downloaded." << endl;
            }
            //If we know the size
            if (this->ResponseType == 1) {
                int receivedByte = -1;
                char buffer;
                int receivedSize = 0;
                cout << "(" << size << " bytes)..." << endl << "0%";
                while (receivedSize < size) {
                    if ((receivedByte = read(Socket, &buffer, 1)) < 0) {
                        this->RequestMessage = "Connection error. Did the connection abort?";
                        cerr << this->RequestMessage << endl;
                        return false;
                    }
                    receivedSize += receivedByte;
                    this->Response += buffer;
                    cout << "\r" << receivedSize * 100 / size << "%" << flush; //Print percentage of downloaded file
                }
                cout << endl;
            }
            //We don't know the size but we now that the transfer is chunked
            if (this->ResponseType == 2) {
                int receivedSize = 0;
                cout << "(chunked)..." << endl;
                while (true) {
                    //Read size of next chunk
                    string HTTPChunkHeader;
                    HTTPChunkHeader = getLine(Socket);
                    int chunkSize = -1;
                    chunkSize = stoul(HTTPChunkHeader, nullptr, 16); //Read the length of the next part (in hexadecimal)
                    if (chunkSize <= 0) {
                        break; //Okay, last chunk downloaded - "Over and out"
                    }
                    //Download the next chunk
                    receivedSize = 0;
                    cout << "Downloading one part (" << chunkSize << " bytes)... " << endl << "0%";
                    while (receivedSize < chunkSize) {
                        char buffer;
                        if (read(Socket, &buffer, 1) <= 0) {
                            this->RequestMessage = "Connection error. Did the connection abort?";
                            cerr << this->RequestMessage << endl;
                            return false;
                        }
                        receivedSize++;
                        this->Response += buffer;
                        cout << "\r" << receivedSize * 100 / chunkSize << "%" << flush;
                    }
                    cout << endl;
                    //Now get the "\r\n" out of the socket
                    for (int i = 0; i < 2; ++i) {
                        char temp;
                        read(Socket, &temp, 1);
                    }
                }
            }
            close(Socket); //We may downloaded something, but now is all over
            this->RequestMessage = "All OK.";
            cout << "Disconnected." << endl;
            return true;
        }
    };
    
    int main() {
        HTTPRequest *HTTP = new HTTPRequest();
        HTTP->setHost("build.X.de");
        HTTP->setPath("/response.php");
        HTTP->setData("POST", "user=simon&project=curl");
        HTTP->perform();
        cout << HTTP->Response << endl;
    
        exit(0);
    
        //const string HTTPrequest = "POST /Mein%20LOGO%202.0.png HTTP/1.1\r\nHost: build.X.de\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 23\r\nConnection: close\r\n\r\nuser=simon&project=curl\r\n";
        HTTP->setHost("build.X.de");
        HTTP->setPath("/Mein LOGO 2.0.png");
        //HTTP->setPath("/10MB.zip");
        //HTTP->setPath("/1GB.zip");
        HTTP->perform();
    
        ofstream FILE;
        FILE.open("FILE.png", ios_base::out);
        FILE.write(HTTP->Response.c_str(), HTTP->Response.length());
        FILE.close();
    
        HTTPRequest *HTTP2 = new HTTPRequest();
        HTTP2->perform();
        cout << HTTP2->Response << endl;
    }
    

    Und hier noch das Antwortpaket von Cloudflare (wie es mein Programm empfängt):

    HTTP-HEAD-ANSWER
    Date: Mon, 29 Jan 2018 11:34:06 GMT
    Content-Type: text/html; charset=UTF-8
    Set-Cookie: __cfduid=d4acb17ae9b84e68521628f52e5fff45c1517225646; expires=Tue, 29-Jan-19 11:34:06 GMT; path=/; domain=.X.de; HttpOnly
    Vary: Accept-Encoding
    X-Content-Type-Options: nosniff
    Server: cloudflare
    CF-RAY: 3e4be15fd2d36451-FRA
    X-Cache: MISS from XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
    X-Cache-Lookup: MISS from XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:3128
    Via: 1.1 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX (squid/3.4.8)
    Connection: close
    
    HTTP-HEAD-ANSWER END
    

    Und hier noch das Antwortpaket (Gecaptured mit Wireshark) von Cloudflare (in der vollständigen Version):

    Frame 86: 66 bytes on wire (528 bits), 66 bytes captured (528 bits) on interface 0
    Ethernet II, Src: Tp-LinkT_b5:3b:2d (f8:d1:11:b5:3b:2d), Dst: IntelCor_f1:bc:33 (e4:f8:9c:f1:bc:33)
    Internet Protocol Version 4, Src: 104.27.181.182, Dst: 10.0.2.201
    Transmission Control Protocol, Src Port: 80, Dst Port: 57950, Seq: 616, Ack: 172, Len: 0
    [3 Reassembled TCP Segments (615 bytes): #84(513), #85(102), #86(0)]
    Hypertext Transfer Protocol
        HTTP/1.1 200 OK\r\n
        Date: Mon, 29 Jan 2018 11:04:05 GMT\r\n
        Content-Type: text/html; charset=UTF-8\r\n
        Set-Cookie: __cfduid=d6485ba93ba59990b87022c050bd6efa71517223845; expires=Tue, 29-Jan-19 11:04:05 GMT; path=/; domain=.X.de; HttpOnly\r\n
        Vary: Accept-Encoding\r\n
        X-Content-Type-Options: nosniff\r\n
        Server: cloudflare\r\n
        CF-RAY: 3e4bb56820bc646f-FRA\r\n
        X-Cache: MISS from XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\r\n
        X-Cache-Lookup: MISS from XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:3128\r\n
        Via: 1.1 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX (squid/3.4.8)\r\n
        Connection: close\r\n
        \r\n
        [HTTP response 1/1]
        [Time since request: 0.395722782 seconds]
        [Request in frame: 78]
        File Data: 102 bytes
    Line-based text data: text/html
        #Request accepted.\n
        #Follwing POST data received:\n
        \n
        Array\n
        (\n
            [user] => simon\n
            [project] => curl\n
        )\n
    

    Also wer eine Idee hat... Danke!



  • Die C++ Frage ist wo?



  • Das kann durchaus vorkommen. Du musst dann solange lesen, bis die Verbindung geschlossen wird.



  • Ein Idiot bleibt ein Idiot. In diesm Falle bin ich dieser Idiot.
    SORRY! 😞

    cout << "HTTP-HEAD-ANSWER END" << endl;
            }
    
            if(size == -1) {
                this->ResponseType = 0;
            }
    
            //Now get the content
            cout << "Downloading ";
    

    Ich hatte vergessen auch den "NoSize"-Fall auszulösen, sollte nicht einmal das HTTP-Header-Packet 'Content-Length' erwähnen! Aua.

    @Techel

    Nein, muss ich nicht, da ich explizit den Server anweise die Verbindung sofort nach Versand der Daten zu kappen. Zudem: Wie wäre sonst das Ende von einem Fehler zu unterscheiden? :p

    Danke an meine DOOFHEIT,
    Simon 🕶


Anmelden zum Antworten