132
|
1 /*******************************************************************************
|
|
2
|
|
3 copyright: Copyright (c) 2004 Kris Bell. All rights reserved
|
|
4
|
|
5 license: BSD style: $(LICENSE)
|
|
6
|
|
7 version: Initial release: April 2004
|
|
8 Outback release: December 2006
|
|
9
|
|
10 author: Kris - original module
|
|
11 author: h3r3tic - fixed a number of Post issues and
|
|
12 bugs in the 'params' construction
|
|
13
|
|
14 Redirection handling guided via
|
|
15 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html
|
|
16
|
|
17 *******************************************************************************/
|
|
18
|
|
19 module tango.net.http.HttpClient;
|
|
20
|
|
21 private import tango.time.Time;
|
|
22
|
|
23 private import tango.io.Buffer;
|
|
24
|
|
25 private import tango.net.Uri,
|
|
26 tango.net.SocketConduit,
|
|
27 tango.net.InternetAddress;
|
|
28
|
|
29 private import tango.net.http.HttpConst,
|
|
30 tango.net.http.HttpParams,
|
|
31 tango.net.http.HttpHeaders,
|
|
32 tango.net.http.HttpTriplet,
|
|
33 tango.net.http.HttpCookies;
|
|
34
|
|
35 private import tango.text.stream.LineIterator;
|
|
36
|
|
37 private import Integer = tango.text.convert.Integer;
|
|
38
|
|
39
|
|
40 /*******************************************************************************
|
|
41
|
|
42 Supports the basic needs of a client making requests of an HTTP
|
|
43 server. The following is an example of how this might be used:
|
|
44
|
|
45 ---
|
|
46 // callback for client reader
|
|
47 void sink (char[] content)
|
|
48 {
|
|
49 Stdout.put (content);
|
|
50 }
|
|
51
|
|
52 // create client for a GET request
|
|
53 auto client = new HttpClient (HttpClient.Get, "http://www.yahoo.com");
|
|
54
|
|
55 // make request
|
|
56 client.open ();
|
|
57
|
|
58 // check return status for validity
|
|
59 if (client.isResponseOK)
|
|
60 {
|
|
61 // extract content length
|
|
62 auto length = client.getResponseHeaders.getInt (HttpHeader.ContentLength, uint.max);
|
|
63
|
|
64 // display all returned headers
|
|
65 Stdout.put (client.getResponseHeaders);
|
|
66
|
|
67 // display remaining content
|
|
68 client.read (&sink, length);
|
|
69 }
|
|
70 else
|
|
71 Stderr.put (client.getResponse);
|
|
72
|
|
73 client.close ();
|
|
74 ---
|
|
75
|
|
76 See modules HttpGet and HttpPost for simple wrappers instead.
|
|
77
|
|
78 *******************************************************************************/
|
|
79
|
|
80 class HttpClient
|
|
81 {
|
|
82 // callback for sending PUT content
|
|
83 alias void delegate (IBuffer) Pump;
|
|
84
|
|
85 // this is struct rather than typedef to avoid compiler bugs
|
|
86 private struct RequestMethod
|
|
87 {
|
|
88 final char[] name;
|
|
89 }
|
|
90
|
|
91 // class members; there's a surprising amount of stuff here!
|
|
92 private Uri uri;
|
|
93 private IBuffer tmp,
|
|
94 input,
|
|
95 output;
|
|
96 private SocketConduit socket;
|
|
97 private RequestMethod method;
|
|
98 private InternetAddress address;
|
|
99 private HttpParams paramsOut;
|
|
100 private HttpHeadersView headersIn;
|
|
101 private HttpHeaders headersOut;
|
|
102 private HttpCookies cookiesOut;
|
|
103 private ResponseLine responseLine;
|
|
104
|
|
105 // default to three second timeout on read operations ...
|
|
106 private TimeSpan timeout = TimeSpan.seconds(3);
|
|
107
|
|
108 // should we perform internal redirection?
|
|
109 private bool doRedirect = true;
|
|
110
|
|
111 // use HTTP v1.0?
|
|
112 private static const char[] DefaultHttpVersion = "HTTP/1.0";
|
|
113
|
|
114 // standard set of request methods ...
|
|
115 static const RequestMethod Get = {"GET"},
|
|
116 Put = {"PUT"},
|
|
117 Head = {"HEAD"},
|
|
118 Post = {"POST"},
|
|
119 Trace = {"TRACE"},
|
|
120 Delete = {"DELETE"},
|
|
121 Options = {"OPTIONS"},
|
|
122 Connect = {"CONNECT"};
|
|
123
|
|
124 /***********************************************************************
|
|
125
|
|
126 Create a client for the given URL. The argument should be
|
|
127 fully qualified with an "http:" or "https:" scheme, or an
|
|
128 explicit port should be provided.
|
|
129
|
|
130 ***********************************************************************/
|
|
131
|
|
132 this (RequestMethod method, char[] url)
|
|
133 {
|
|
134 this (method, new Uri(url));
|
|
135 }
|
|
136
|
|
137 /***********************************************************************
|
|
138
|
|
139 Create a client with the provided Uri instance. The Uri should
|
|
140 be fully qualified with an "http:" or "https:" scheme, or an
|
|
141 explicit port should be provided.
|
|
142
|
|
143 ***********************************************************************/
|
|
144
|
|
145 this (RequestMethod method, Uri uri)
|
|
146 {
|
|
147 this.uri = uri;
|
|
148 this.method = method;
|
|
149
|
|
150 responseLine = new ResponseLine;
|
|
151 headersIn = new HttpHeadersView;
|
|
152
|
|
153 tmp = new Buffer (1024 * 4);
|
|
154 paramsOut = new HttpParams (new Buffer (1024 * 4));
|
|
155 headersOut = new HttpHeaders (new Buffer (1024 * 4));
|
|
156 cookiesOut = new HttpCookies (headersOut);
|
|
157
|
|
158 // decode the host name (may take a second or two)
|
|
159 auto host = uri.getHost ();
|
|
160 if (host)
|
|
161 address = new InternetAddress (host, uri.getValidPort);
|
|
162 else
|
|
163 responseLine.error ("invalid url provided to HttpClient ctor");
|
|
164 }
|
|
165
|
|
166 /***********************************************************************
|
|
167
|
|
168 Get the current input headers, as returned by the host request.
|
|
169
|
|
170 ***********************************************************************/
|
|
171
|
|
172 HttpHeadersView getResponseHeaders()
|
|
173 {
|
|
174 return headersIn;
|
|
175 }
|
|
176
|
|
177 /***********************************************************************
|
|
178
|
|
179 Gain access to the request headers. Use this to add whatever
|
|
180 headers are required for a request.
|
|
181
|
|
182 ***********************************************************************/
|
|
183
|
|
184 HttpHeaders getRequestHeaders()
|
|
185 {
|
|
186 return headersOut;
|
|
187 }
|
|
188
|
|
189 /***********************************************************************
|
|
190
|
|
191 Gain access to the request parameters. Use this to add x=y
|
|
192 style parameters to the request. These will be appended to
|
|
193 the request assuming the original Uri does not contain any
|
|
194 of its own.
|
|
195
|
|
196 ***********************************************************************/
|
|
197
|
|
198 HttpParams getRequestParams()
|
|
199 {
|
|
200 return paramsOut;
|
|
201 }
|
|
202
|
|
203 /***********************************************************************
|
|
204
|
|
205 Return the Uri associated with this client
|
|
206
|
|
207 ***********************************************************************/
|
|
208
|
|
209 UriView getUri()
|
|
210 {
|
|
211 return uri;
|
|
212 }
|
|
213
|
|
214 /***********************************************************************
|
|
215
|
|
216 Return the response-line for the latest request. This takes
|
|
217 the form of "version status reason" as defined in the HTTP
|
|
218 RFC.
|
|
219
|
|
220 ***********************************************************************/
|
|
221
|
|
222 ResponseLine getResponse()
|
|
223 {
|
|
224 return responseLine;
|
|
225 }
|
|
226
|
|
227 /***********************************************************************
|
|
228
|
|
229 Return the HTTP status code set by the remote server
|
|
230
|
|
231 ***********************************************************************/
|
|
232
|
|
233 int getStatus()
|
|
234 {
|
|
235 return responseLine.getStatus;
|
|
236 }
|
|
237
|
|
238 /***********************************************************************
|
|
239
|
|
240 Return whether the response was OK or not
|
|
241
|
|
242 ***********************************************************************/
|
|
243
|
|
244 bool isResponseOK()
|
|
245 {
|
|
246 return getStatus is HttpResponseCode.OK;
|
|
247 }
|
|
248
|
|
249 /***********************************************************************
|
|
250
|
|
251 Add a cookie to the outgoing headers
|
|
252
|
|
253 ***********************************************************************/
|
|
254
|
|
255 void addCookie (Cookie cookie)
|
|
256 {
|
|
257 cookiesOut.add (cookie);
|
|
258 }
|
|
259
|
|
260 /***********************************************************************
|
|
261
|
|
262 Close all resources used by a request. You must invoke this
|
|
263 between successive open() calls.
|
|
264
|
|
265 ***********************************************************************/
|
|
266
|
|
267 void close ()
|
|
268 {
|
|
269 if (socket)
|
|
270 {
|
|
271 socket.shutdown;
|
|
272 socket.detach;
|
|
273 socket = null;
|
|
274 }
|
|
275 }
|
|
276
|
|
277 /***********************************************************************
|
|
278
|
|
279 Reset the client such that it is ready for a new request.
|
|
280
|
|
281 ***********************************************************************/
|
|
282
|
|
283 void reset ()
|
|
284 {
|
|
285 headersIn.reset;
|
|
286 headersOut.reset;
|
|
287 paramsOut.reset;
|
|
288 }
|
|
289
|
|
290 /***********************************************************************
|
|
291
|
|
292 enable/disable the internal redirection suppport
|
|
293
|
|
294 ***********************************************************************/
|
|
295
|
|
296 void enableRedirect (bool yes)
|
|
297 {
|
|
298 doRedirect = yes;
|
|
299 }
|
|
300
|
|
301 /***********************************************************************
|
|
302
|
|
303 set timeout period for read operation
|
|
304
|
|
305 ***********************************************************************/
|
|
306
|
|
307 void setTimeout (TimeSpan interval)
|
|
308 {
|
|
309 timeout = interval;
|
|
310 }
|
|
311
|
|
312 /**
|
|
313 * Deprecated: use setTimeout(TimeSpan) instead
|
|
314 */
|
|
315 deprecated void setTimeout(double interval)
|
|
316 {
|
|
317 setTimeout(TimeSpan.interval(interval));
|
|
318 }
|
|
319
|
|
320 /***********************************************************************
|
|
321
|
|
322 Overridable method to create a Socket. You may find a need
|
|
323 to override this for some purpose; perhaps to add input or
|
|
324 output filters.
|
|
325
|
|
326 ***********************************************************************/
|
|
327
|
|
328 protected SocketConduit createSocket ()
|
|
329 {
|
|
330 return new SocketConduit;
|
|
331 }
|
|
332
|
|
333 /***********************************************************************
|
|
334
|
|
335 Make a request for the resource specified via the constructor,
|
|
336 using the specified timeout period (in milli-seconds).The
|
|
337 return value represents the input buffer, from which all
|
|
338 returned headers and content may be accessed.
|
|
339
|
|
340 ***********************************************************************/
|
|
341
|
|
342 IBuffer open (IBuffer buffer = null)
|
|
343 {
|
|
344 return open (method, null, buffer);
|
|
345 }
|
|
346
|
|
347 /***********************************************************************
|
|
348
|
|
349 Make a request for the resource specified via the constructor,
|
|
350 using a callback for pumping additional data to the host. This
|
|
351 defaults to a three-second timeout period. The return value
|
|
352 represents the input buffer, from which all returned headers
|
|
353 and content may be accessed.
|
|
354
|
|
355 ***********************************************************************/
|
|
356
|
|
357 IBuffer open (Pump pump, IBuffer buffer = null)
|
|
358 {
|
|
359 return open (method, pump, buffer);
|
|
360 }
|
|
361
|
|
362 /***********************************************************************
|
|
363
|
|
364 Make a request for the resource specified via the constructor
|
|
365 using the specified timeout period (in micro-seconds), and a
|
|
366 user-defined callback for pumping additional data to the host.
|
|
367 The callback would be used when uploading data during a 'put'
|
|
368 operation (or equivalent). The return value represents the
|
|
369 input buffer, from which all returned headers and content may
|
|
370 be accessed.
|
|
371
|
|
372 Note that certain request-headers may generated automatically
|
|
373 if they are not present. These include a Host header and, in
|
|
374 the case of Post, both ContentType & ContentLength for a query
|
|
375 type of request. The latter two are *not* produced for Post
|
|
376 requests with 'pump' specified ~ when using 'pump' to output
|
|
377 additional content, you must explicitly set your own headers.
|
|
378
|
|
379 Note also that IOException instances may be thrown. These
|
|
380 should be caught by the client to ensure a close() operation
|
|
381 is always performed
|
|
382
|
|
383 ***********************************************************************/
|
|
384
|
|
385 private IBuffer open (RequestMethod method, Pump pump, IBuffer input)
|
|
386 {
|
|
387 // create socket, and connect it
|
|
388 socket = createSocket;
|
|
389 socket.setTimeout (timeout);
|
|
390 socket.connect (address);
|
|
391
|
|
392 // create buffers for input and output
|
|
393 if (input)
|
|
394 input.clear, input.setConduit (socket);
|
|
395 else
|
|
396 input = new Buffer (socket);
|
|
397 output = new Buffer (socket);
|
|
398
|
|
399 // save for read() method
|
|
400 this.input = input;
|
|
401
|
|
402 // setup a Host header
|
|
403 if (headersOut.get (HttpHeader.Host, null) is null)
|
|
404 headersOut.add (HttpHeader.Host, uri.getHost);
|
|
405
|
|
406 // http/1.0 needs connection:close
|
|
407 headersOut.add (HttpHeader.Connection, "close");
|
|
408
|
|
409 // attach/extend query parameters if user has added some
|
|
410 tmp.clear;
|
|
411 auto query = uri.extendQuery (paramsOut.formatTokens(tmp, "&"));
|
|
412
|
|
413 // patch request path?
|
|
414 auto path = uri.getPath;
|
|
415 if (path.length is 0)
|
|
416 path = "/";
|
|
417
|
|
418 // format encoded request
|
|
419 output (method.name) (" "), uri.encode (&output.consume, path, uri.IncPath);
|
|
420
|
|
421 // should we emit query as part of request line?
|
|
422 if (query.length)
|
|
423 if (method is Get)
|
|
424 output ("?"), uri.encode (&output.consume, query, uri.IncQueryAll);
|
|
425 else
|
|
426 if (method is Post && pump.funcptr is null)
|
|
427 {
|
|
428 // we're POSTing query text - add default info
|
|
429 if (headersOut.get (HttpHeader.ContentType, null) is null)
|
|
430 headersOut.add (HttpHeader.ContentType, "application/x-www-form-urlencoded");
|
|
431
|
|
432 if (headersOut.get (HttpHeader.ContentLength, null) is null)
|
|
433 headersOut.addInt (HttpHeader.ContentLength, query.length);
|
|
434 }
|
|
435
|
|
436 // complete the request line, and emit headers too
|
|
437 output (" ") (DefaultHttpVersion) (HttpConst.Eol);
|
|
438
|
|
439 headersOut.produce (&output.consume, HttpConst.Eol);
|
|
440 output (HttpConst.Eol);
|
|
441
|
|
442 // user has additional data to send?
|
|
443 if (pump.funcptr)
|
|
444 pump (output);
|
|
445 else
|
|
446 // send encoded POST query instead?
|
|
447 if (method is Post && query.length)
|
|
448 uri.encode (&output.consume, query, uri.IncQueryAll);
|
|
449
|
|
450 // send entire request
|
|
451 output.flush;
|
|
452
|
|
453 // Token for initial parsing of input header lines
|
|
454 auto line = new LineIterator!(char) (input);
|
|
455
|
|
456 // skip any blank lines
|
|
457 while (line.next && line.get.length is 0)
|
|
458 {}
|
|
459
|
|
460 // throw if we experienced a timeout
|
|
461 if (socket.hadTimeout)
|
|
462 responseLine.error ("response timeout");
|
|
463
|
|
464 // is this a bogus request?
|
|
465 if (line.get.length is 0)
|
|
466 responseLine.error ("truncated response");
|
|
467
|
|
468 // read response line
|
|
469 responseLine.parse (line.get);
|
|
470
|
|
471 // parse incoming headers
|
|
472 headersIn.reset.parse (input);
|
|
473
|
|
474 // check for redirection
|
|
475 if (doRedirect)
|
|
476 switch (responseLine.getStatus)
|
|
477 {
|
|
478 case HttpResponseCode.SeeOther:
|
|
479 case HttpResponseCode.MovedPermanently:
|
|
480 case HttpResponseCode.MovedTemporarily:
|
|
481 case HttpResponseCode.TemporaryRedirect:
|
|
482 // drop this connection
|
|
483 close;
|
|
484
|
|
485 // remove any existing Host header
|
|
486 headersOut.remove (HttpHeader.Host);
|
|
487
|
|
488 // parse redirected uri
|
|
489 auto redirect = headersIn.get (HttpHeader.Location, "[missing url]");
|
|
490 uri.relParse (redirect);
|
|
491
|
|
492 // decode the host name (may take a second or two)
|
|
493 auto host = uri.getHost();
|
|
494 if (host)
|
|
495 address = new InternetAddress (uri.getHost, uri.getValidPort);
|
|
496 else
|
|
497 responseLine.error ("redirect has invalid url: "~redirect);
|
|
498
|
|
499 // figure out what to do
|
|
500 if (method is Get || method is Head)
|
|
501 return open (method, pump, input);
|
|
502 else
|
|
503 if (method is Post)
|
|
504 return redirectPost (pump, input, responseLine.getStatus);
|
|
505 else
|
|
506 responseLine.error ("unexpected redirect for method "~method.name);
|
|
507 default:
|
|
508 break;
|
|
509 }
|
|
510
|
|
511 // return the input buffer
|
|
512 return input;
|
|
513 }
|
|
514
|
|
515 /***********************************************************************
|
|
516
|
|
517 Read the content from the returning input stream, up to a
|
|
518 maximum length, and pass content to the given sink delegate
|
|
519 as it arrives.
|
|
520
|
|
521 Exits when length bytes have been processed, or an Eof is
|
|
522 seen on the stream.
|
|
523
|
|
524 ***********************************************************************/
|
|
525
|
|
526 void read (void delegate (void[]) sink, long len = long.max)
|
|
527 {
|
|
528 while (len > 0)
|
|
529 {
|
|
530 auto content = input.slice;
|
|
531 if ((len -= content.length) < 0)
|
|
532 {
|
|
533 content = content [0 .. $ + cast(size_t) len];
|
|
534 sink (content);
|
|
535 input.skip (content.length);
|
|
536 }
|
|
537 else
|
|
538 {
|
|
539 sink (content);
|
|
540 input.clear;
|
|
541 if (input.fill(input.input) is socket.Eof)
|
|
542 break;
|
|
543 }
|
|
544 }
|
|
545 }
|
|
546
|
|
547 /***********************************************************************
|
|
548
|
|
549 Handle redirection of Post
|
|
550
|
|
551 Guidance for the default behaviour came from this page:
|
|
552 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html
|
|
553
|
|
554 ***********************************************************************/
|
|
555
|
|
556 IBuffer redirectPost (Pump pump, IBuffer input, int status)
|
|
557 {
|
|
558 switch (status)
|
|
559 {
|
|
560 // use Get method to complete the Post
|
|
561 case HttpResponseCode.SeeOther:
|
|
562 case HttpResponseCode.MovedTemporarily:
|
|
563
|
|
564 // remove POST headers first!
|
|
565 headersOut.remove (HttpHeader.ContentLength);
|
|
566 headersOut.remove (HttpHeader.ContentType);
|
|
567 paramsOut.reset;
|
|
568 return open (Get, null, input);
|
|
569
|
|
570 // try entire Post again, if user say OK
|
|
571 case HttpResponseCode.MovedPermanently:
|
|
572 case HttpResponseCode.TemporaryRedirect:
|
|
573 if (canRepost (status))
|
|
574 return open (this.method, pump, input);
|
|
575 // fall through!
|
|
576
|
|
577 default:
|
|
578 responseLine.error ("Illegal redirection of Post");
|
|
579 }
|
|
580 return null;
|
|
581 }
|
|
582
|
|
583 /***********************************************************************
|
|
584
|
|
585 Handle user-notification of Post redirection. This should
|
|
586 be overridden appropriately.
|
|
587
|
|
588 Guidance for the default behaviour came from this page:
|
|
589 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html
|
|
590
|
|
591 ***********************************************************************/
|
|
592
|
|
593 bool canRepost (uint status)
|
|
594 {
|
|
595 return false;
|
|
596 }
|
|
597 }
|
|
598
|
|
599
|
|
600 /******************************************************************************
|
|
601
|
|
602 Class to represent an HTTP response-line
|
|
603
|
|
604 ******************************************************************************/
|
|
605
|
|
606 private class ResponseLine : HttpTriplet
|
|
607 {
|
|
608 private char[] vers,
|
|
609 reason;
|
|
610 private int status;
|
|
611
|
|
612 /**********************************************************************
|
|
613
|
|
614 test the validity of these tokens
|
|
615
|
|
616 **********************************************************************/
|
|
617
|
|
618 void test ()
|
|
619 {
|
|
620 vers = tokens[0];
|
|
621 reason = tokens[2];
|
|
622 status = cast(int) Integer.convert (tokens[1]);
|
|
623 if (status is 0)
|
|
624 {
|
|
625 status = cast(int) Integer.convert (tokens[2]);
|
|
626 if (status is 0)
|
|
627 error ("Invalid HTTP response: '"~tokens[0]~"' '"~tokens[1]~"' '" ~tokens[2] ~"'");
|
|
628 }
|
|
629 }
|
|
630
|
|
631 /**********************************************************************
|
|
632
|
|
633 Return HTTP version
|
|
634
|
|
635 **********************************************************************/
|
|
636
|
|
637 char[] getVersion ()
|
|
638 {
|
|
639 return vers;
|
|
640 }
|
|
641
|
|
642 /**********************************************************************
|
|
643
|
|
644 Return reason text
|
|
645
|
|
646 **********************************************************************/
|
|
647
|
|
648 char[] getReason ()
|
|
649 {
|
|
650 return reason;
|
|
651 }
|
|
652
|
|
653 /**********************************************************************
|
|
654
|
|
655 Return status integer
|
|
656
|
|
657 **********************************************************************/
|
|
658
|
|
659 int getStatus ()
|
|
660 {
|
|
661 return status;
|
|
662 }
|
|
663 }
|