Frage zu HTTP-Header-Antwort
-
Hallo,
ich schreibe zur Zeit mal wieder an meinem HTTP-client in C++ rum. Heute habe ich das Problem bekommen, dass bei dem Antwort-Paket, welches ich von meiner Seite angefordert habe, weder die 'chunked transfer'-Option, noch die 'content-length' angegeben wurde, weshalb mein Programm nun versagt. Meine Frage ist, ob dies nur ein Server-Fehler (Cloudflare) ist, oder ob ich eine Möglichkeit übersehen habe, wie man dennoch solche Pakete Pharsen kann.Hier folgt nun der Code; ich möchte noch erwähnen, dass der "Fehler" in Zeile 281 auftritt. Und ja, ich habe den ganzen Code drangehängt, damit man den Fehler selbst sehen kann.
#include <iostream> #include <fstream> #include <cstdlib> #include <sys/socket.h> // socket(), connect() #include <arpa/inet.h> // sockaddr_in #include <netdb.h> // gethostbyname(), hostent #include <unistd.h> #define DEBUG_FLAG true using namespace std; class HTTPRequest { public: string RequestHost; string RequestPath; string RequestDataType; string RequestData; string Response; string ResponseProtocol; int ResponseCode; string ResponseCodeMessage; string RequestMessage; string ResponseDate; string ResponseVia; string ResponseDataType; private: bool Initialized = false; hostent* RequestHostResolved; int ResponseType = 0; // 0 = Unknown Size , 1 = Normal , 2 = Chunked string getLine(int socket) { string line; while (true) { char c; if (read(socket, &c, 1) <= 0) { this->RequestMessage = "Error on receiving"; cerr << this->RequestMessage << endl; return ""; } if (c == '\n') { return line; } line += c; } } void replaceAll(string* str, string from, string to) { string temp; for (int i = 0; i < str->length(); i++) { if (str->c_str()[i] != ' ') { temp += str->c_str()[i]; } else { temp += '%'; temp += '2'; temp += '0'; } } *str = temp; } public: HTTPRequest() { this->setHost("example.com"); this->setPath("/"); this->setDataType("Undefined"); this->setData(""); this->Response = ""; this->ResponseProtocol = "Unknown"; this->ResponseCode = 000; this->ResponseCodeMessage = ""; this->RequestMessage = ""; this->ResponseVia = "Not defined"; this->ResponseDataType = "Not defined"; this->Initialized = true; } void setPath(string GivenPath) { this->RequestPath = GivenPath; } void setData(string GivenData) { this->RequestData = GivenData; } void setData(string GivenDataType, string GivenData) { this->RequestData = GivenData; this->RequestDataType = GivenDataType; } void setDataType(string GivenDataType) { this->RequestDataType = GivenDataType; } bool setHost(string GivenHost) { this->Initialized = true; this->RequestHost = GivenHost; cout << "Resolving '" << this->RequestHost << "'" << endl; this->RequestHostResolved = gethostbyname(this->RequestHost.c_str()); if (RequestHostResolved == NULL) { this->RequestMessage = "Can't resolve hostname!"; cerr << this->RequestMessage << endl; return false; } if (RequestHostResolved->h_addrtype != AF_INET) { this->RequestMessage = "ERROR. Only IPv4 is supported (Unknown type)!"; cerr << this->RequestMessage << endl; return false; } if (RequestHostResolved->h_length != 4) { this->RequestMessage = "ERROR. Only IPv4 is supported (Wrong type)!"; cerr << this->RequestMessage << endl; return false; } if (DEBUG_FLAG) { cout << this->RequestHost << " resolved." << endl; } } bool perform() { if (!Initialized) { this->RequestMessage = "Error. The request is not fully initialized!"; cerr << this->RequestMessage << endl; return false; } this->Response = ""; cout << "Connecting to '" << this->RequestHost << "'" << endl; //Ask for a socket to communicate int Socket = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); if (Socket == -1) { this->RequestMessage = "Can't get a free socket!"; cerr << this->RequestMessage << endl; return false; } //Bind adresses and go! sockaddr_in service_addr; service_addr.sin_family = AF_INET; service_addr.sin_addr.s_addr = inet_addr(inet_ntoa(*(struct in_addr*) (RequestHostResolved->h_addr_list[0]))); service_addr.sin_port = htons(80); //Use default HTTPport if (connect(Socket, (struct sockaddr *) &service_addr, sizeof (service_addr)) < 0) { this->RequestMessage = "Connection failed!"; cerr << this->RequestMessage << endl; return false; } else { if (DEBUG_FLAG) { cout << "Successfully connected." << endl; } } replaceAll(&this->RequestPath, " ", "%20"); replaceAll(&this->RequestData, " ", "%20"); //Generate the HTTP-package string HTTPrequest = "\n"; if (this->RequestDataType == "POST") { HTTPrequest.clear(); HTTPrequest.append("POST "); HTTPrequest.append(this->RequestPath.c_str()); HTTPrequest.append(" HTTP/1.1\r\nHost: "); HTTPrequest.append(this->RequestHost.c_str()); HTTPrequest.append("\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: "); HTTPrequest.append(to_string(this->RequestData.length())); HTTPrequest.append("\r\nConnection: close\r\n\r\n"); HTTPrequest.append(this->RequestData.c_str()); HTTPrequest.append("\r\n"); //HTTPrequest = "POST /response.php HTTP/1.1\r\nHost: example.com\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 23\r\nConnection: close\r\n\r\nuser=simon&project=nope\r\n"; } if (this->RequestDataType == "GET") { HTTPrequest.clear(); HTTPrequest.append("GET "); HTTPrequest.append(this->RequestPath.c_str()); HTTPrequest.append("?"); HTTPrequest.append(this->RequestData); HTTPrequest.append(" HTTP/1.1\r\nHost: "); HTTPrequest.append(this->RequestHost); HTTPrequest.append("\r\nConnection: close\r\n\r\n"); //HTTPrequest = "GET /response.php?user=simon&project=nope HTTP/1.1\r\nHost: example.com\r\nConnection: close\r\n\r\n"; } if (this->RequestDataType == "" || this->RequestDataType == "Undefined") { HTTPrequest.clear(); HTTPrequest.append("GET "); HTTPrequest.append(this->RequestPath.c_str()); HTTPrequest.append(" HTTP/1.1\r\nHost: "); HTTPrequest.append(this->RequestHost); HTTPrequest.append("\r\nConnection: close\r\n\r\n"); //HTTPrequest = "GET /response.php HTTP/1.1\r\nHost: example.com\r\nConnection: close\r\n\r\n"; } if (DEBUG_FLAG) { cout << "\nHTTP-REQUEST\n" << HTTPrequest << "HTTP-REQUEST END\n" << endl; } if (write(Socket, HTTPrequest.c_str(), HTTPrequest.size()) != HTTPrequest.size()) { this->RequestMessage = "Connection error. Did the connection abort?"; cerr << this->RequestMessage << endl; return false; } //Pharse the header package for information this->ResponseCode = 100; string HTTPHeader; while (ResponseCode == 100) { HTTPHeader += getLine(Socket); this->ResponseProtocol = HTTPHeader.substr(0, HTTPHeader.find(' ')); this->ResponseCode = atoi(HTTPHeader.substr(HTTPHeader.find(' ') + 1, 3).c_str()); if (this->ResponseCode == 100) { HTTPHeader += getLine(Socket); //Ignore the empty line after 'Continue' (Code 100) } } if (DEBUG_FLAG) { cout << "Protocol used: " << this->ResponseProtocol << endl; cout << "Response code: " << this->ResponseCode << endl; } if (this->ResponseCode != 200) { this->ResponseCodeMessage = HTTPHeader; return false; } int size = -1; if (DEBUG_FLAG) { cout << "\nHTTP-HEAD-ANSWER\n"; } this->ResponseType = -1; while (true) { HTTPHeader = getLine(Socket); if (DEBUG_FLAG) { cout << HTTPHeader << endl; } if (HTTPHeader == "\r") //Header ended (One clean line) { break; } string option = HTTPHeader.substr(0, HTTPHeader.find(':') + 1); //The option to set if (option == "Content-Length:") { size = atoi(HTTPHeader.substr(HTTPHeader.find(':') + 2, (HTTPHeader.length() - HTTPHeader.find(':') + 2)).c_str()); if (size == -1) { this->ResponseType = 0; } else { this->ResponseType = 1; } } if (option == "Transfer-Encoding:") { string transferEncoding; transferEncoding = HTTPHeader.substr(HTTPHeader.find(':') + 2, (HTTPHeader.length() - HTTPHeader.find(':') + 2)); if (transferEncoding == "chunked\r") { this->ResponseType = 2; } } if (option == "Date:") { this->ResponseDate = HTTPHeader.substr(HTTPHeader.find(':') + 2, (HTTPHeader.length() - HTTPHeader.find(':') + 2)); } if (option == "Via:") { this->ResponseVia = HTTPHeader.substr(HTTPHeader.find(':') + 2, (HTTPHeader.length() - HTTPHeader.find(':') + 2)); } if (option == "Content-Type:") { this->ResponseDataType = HTTPHeader.substr(HTTPHeader.find(':') + 2, (HTTPHeader.length() - HTTPHeader.find(':') + 2)); } } if (DEBUG_FLAG) { cout << "HTTP-HEAD-ANSWER END" << endl; } //Now get the content cout << "Downloading "; //If no size or other options defined if(this->ResponseType == -1) { this->RequestMessage = "Can't download the requested content. There was no size given and no 'chunked' option set!"; cerr << this->RequestMessage << endl; return false; } //If not size is defined if (this->ResponseType == 0) { int receivedBytes = -1; cout << "(Unknown filesize)..." << endl; char buffer = '\0'; while (receivedBytes != 0) //If read() return we reached end of file { if ((receivedBytes = read(Socket, &buffer, 1)) < 0) { this->RequestMessage = "Connection error. Did the connection abort?"; cerr << this->RequestMessage << endl; return false; } this->Response += buffer; if(this->Response.length() % 1024 == 0) { cout << "."; //One point for each KiB } } cout << endl << this->Response.length() << " bytes downloaded." << endl; } //If we know the size if (this->ResponseType == 1) { int receivedByte = -1; char buffer; int receivedSize = 0; cout << "(" << size << " bytes)..." << endl << "0%"; while (receivedSize < size) { if ((receivedByte = read(Socket, &buffer, 1)) < 0) { this->RequestMessage = "Connection error. Did the connection abort?"; cerr << this->RequestMessage << endl; return false; } receivedSize += receivedByte; this->Response += buffer; cout << "\r" << receivedSize * 100 / size << "%" << flush; //Print percentage of downloaded file } cout << endl; } //We don't know the size but we now that the transfer is chunked if (this->ResponseType == 2) { int receivedSize = 0; cout << "(chunked)..." << endl; while (true) { //Read size of next chunk string HTTPChunkHeader; HTTPChunkHeader = getLine(Socket); int chunkSize = -1; chunkSize = stoul(HTTPChunkHeader, nullptr, 16); //Read the length of the next part (in hexadecimal) if (chunkSize <= 0) { break; //Okay, last chunk downloaded - "Over and out" } //Download the next chunk receivedSize = 0; cout << "Downloading one part (" << chunkSize << " bytes)... " << endl << "0%"; while (receivedSize < chunkSize) { char buffer; if (read(Socket, &buffer, 1) <= 0) { this->RequestMessage = "Connection error. Did the connection abort?"; cerr << this->RequestMessage << endl; return false; } receivedSize++; this->Response += buffer; cout << "\r" << receivedSize * 100 / chunkSize << "%" << flush; } cout << endl; //Now get the "\r\n" out of the socket for (int i = 0; i < 2; ++i) { char temp; read(Socket, &temp, 1); } } } close(Socket); //We may downloaded something, but now is all over this->RequestMessage = "All OK."; cout << "Disconnected." << endl; return true; } }; int main() { HTTPRequest *HTTP = new HTTPRequest(); HTTP->setHost("build.X.de"); HTTP->setPath("/response.php"); HTTP->setData("POST", "user=simon&project=curl"); HTTP->perform(); cout << HTTP->Response << endl; exit(0); //const string HTTPrequest = "POST /Mein%20LOGO%202.0.png HTTP/1.1\r\nHost: build.X.de\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 23\r\nConnection: close\r\n\r\nuser=simon&project=curl\r\n"; HTTP->setHost("build.X.de"); HTTP->setPath("/Mein LOGO 2.0.png"); //HTTP->setPath("/10MB.zip"); //HTTP->setPath("/1GB.zip"); HTTP->perform(); ofstream FILE; FILE.open("FILE.png", ios_base::out); FILE.write(HTTP->Response.c_str(), HTTP->Response.length()); FILE.close(); HTTPRequest *HTTP2 = new HTTPRequest(); HTTP2->perform(); cout << HTTP2->Response << endl; }
Und hier noch das Antwortpaket von Cloudflare (wie es mein Programm empfängt):
HTTP-HEAD-ANSWER Date: Mon, 29 Jan 2018 11:34:06 GMT Content-Type: text/html; charset=UTF-8 Set-Cookie: __cfduid=d4acb17ae9b84e68521628f52e5fff45c1517225646; expires=Tue, 29-Jan-19 11:34:06 GMT; path=/; domain=.X.de; HttpOnly Vary: Accept-Encoding X-Content-Type-Options: nosniff Server: cloudflare CF-RAY: 3e4be15fd2d36451-FRA X-Cache: MISS from XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX X-Cache-Lookup: MISS from XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:3128 Via: 1.1 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX (squid/3.4.8) Connection: close HTTP-HEAD-ANSWER END
Und hier noch das Antwortpaket (Gecaptured mit Wireshark) von Cloudflare (in der vollständigen Version):
Frame 86: 66 bytes on wire (528 bits), 66 bytes captured (528 bits) on interface 0 Ethernet II, Src: Tp-LinkT_b5:3b:2d (f8:d1:11:b5:3b:2d), Dst: IntelCor_f1:bc:33 (e4:f8:9c:f1:bc:33) Internet Protocol Version 4, Src: 104.27.181.182, Dst: 10.0.2.201 Transmission Control Protocol, Src Port: 80, Dst Port: 57950, Seq: 616, Ack: 172, Len: 0 [3 Reassembled TCP Segments (615 bytes): #84(513), #85(102), #86(0)] Hypertext Transfer Protocol HTTP/1.1 200 OK\r\n Date: Mon, 29 Jan 2018 11:04:05 GMT\r\n Content-Type: text/html; charset=UTF-8\r\n Set-Cookie: __cfduid=d6485ba93ba59990b87022c050bd6efa71517223845; expires=Tue, 29-Jan-19 11:04:05 GMT; path=/; domain=.X.de; HttpOnly\r\n Vary: Accept-Encoding\r\n X-Content-Type-Options: nosniff\r\n Server: cloudflare\r\n CF-RAY: 3e4bb56820bc646f-FRA\r\n X-Cache: MISS from XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\r\n X-Cache-Lookup: MISS from XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX:3128\r\n Via: 1.1 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX (squid/3.4.8)\r\n Connection: close\r\n \r\n [HTTP response 1/1] [Time since request: 0.395722782 seconds] [Request in frame: 78] File Data: 102 bytes Line-based text data: text/html #Request accepted.\n #Follwing POST data received:\n \n Array\n (\n [user] => simon\n [project] => curl\n )\n
Also wer eine Idee hat... Danke!
-
Die C++ Frage ist wo?
-
Das kann durchaus vorkommen. Du musst dann solange lesen, bis die Verbindung geschlossen wird.
-
Ein Idiot bleibt ein Idiot. In diesm Falle bin ich dieser Idiot.
SORRY!cout << "HTTP-HEAD-ANSWER END" << endl; } if(size == -1) { this->ResponseType = 0; } //Now get the content cout << "Downloading ";
Ich hatte vergessen auch den "NoSize"-Fall auszulösen, sollte nicht einmal das HTTP-Header-Packet 'Content-Length' erwähnen! Aua.
Nein, muss ich nicht, da ich explizit den Server anweise die Verbindung sofort nach Versand der Daten zu kappen. Zudem: Wie wäre sonst das Ende von einem Fehler zu unterscheiden? :p
Danke an meine DOOFHEIT,
Simon