Mercurial > projects > ldc

comparison tango/tango/net/http/HttpClient.d @ 132:1700239cab2e trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[svn r136] MAJOR UNSTABLE UPDATE!!! Initial commit after moving to Tango instead of Phobos. Lots of bugfixes... This build is not suitable for most things.
author lindquist
date Fri, 11 Jan 2008 17:57:40 +0100
parents
children
comparison
equal deleted inserted replaced
131:5825d48b27d1 132:1700239cab2e
1 /*******************************************************************************
2
3 copyright: Copyright (c) 2004 Kris Bell. All rights reserved
4
5 license: BSD style: $(LICENSE)
6
7 version: Initial release: April 2004
8 Outback release: December 2006
9
10 author: Kris - original module
11 author: h3r3tic - fixed a number of Post issues and
12 bugs in the 'params' construction
13
14 Redirection handling guided via
15 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html
16
17 *******************************************************************************/
18
19 module tango.net.http.HttpClient;
20
21 private import tango.time.Time;
22
23 private import tango.io.Buffer;
24
25 private import tango.net.Uri,
26 tango.net.SocketConduit,
27 tango.net.InternetAddress;
28
29 private import tango.net.http.HttpConst,
30 tango.net.http.HttpParams,
31 tango.net.http.HttpHeaders,
32 tango.net.http.HttpTriplet,
33 tango.net.http.HttpCookies;
34
35 private import tango.text.stream.LineIterator;
36
37 private import Integer = tango.text.convert.Integer;
38
39
40 /*******************************************************************************
41
42 Supports the basic needs of a client making requests of an HTTP
43 server. The following is an example of how this might be used:
44
45 ---
46 // callback for client reader
47 void sink (char[] content)
48 {
49 Stdout.put (content);
50 }
51
52 // create client for a GET request
53 auto client = new HttpClient (HttpClient.Get, "http://www.yahoo.com");
54
55 // make request
56 client.open ();
57
58 // check return status for validity
59 if (client.isResponseOK)
60 {
61 // extract content length
62 auto length = client.getResponseHeaders.getInt (HttpHeader.ContentLength, uint.max);
63
64 // display all returned headers
65 Stdout.put (client.getResponseHeaders);
66
67 // display remaining content
68 client.read (&sink, length);
69 }
70 else
71 Stderr.put (client.getResponse);
72
73 client.close ();
74 ---
75
76 See modules HttpGet and HttpPost for simple wrappers instead.
77
78 *******************************************************************************/
79
80 class HttpClient
81 {
82 // callback for sending PUT content
83 alias void delegate (IBuffer) Pump;
84
85 // this is struct rather than typedef to avoid compiler bugs
86 private struct RequestMethod
87 {
88 final char[] name;
89 }
90
91 // class members; there's a surprising amount of stuff here!
92 private Uri uri;
93 private IBuffer tmp,
94 input,
95 output;
96 private SocketConduit socket;
97 private RequestMethod method;
98 private InternetAddress address;
99 private HttpParams paramsOut;
100 private HttpHeadersView headersIn;
101 private HttpHeaders headersOut;
102 private HttpCookies cookiesOut;
103 private ResponseLine responseLine;
104
105 // default to three second timeout on read operations ...
106 private TimeSpan timeout = TimeSpan.seconds(3);
107
108 // should we perform internal redirection?
109 private bool doRedirect = true;
110
111 // use HTTP v1.0?
112 private static const char[] DefaultHttpVersion = "HTTP/1.0";
113
114 // standard set of request methods ...
115 static const RequestMethod Get = {"GET"},
116 Put = {"PUT"},
117 Head = {"HEAD"},
118 Post = {"POST"},
119 Trace = {"TRACE"},
120 Delete = {"DELETE"},
121 Options = {"OPTIONS"},
122 Connect = {"CONNECT"};
123
124 /***********************************************************************
125
126 Create a client for the given URL. The argument should be
127 fully qualified with an "http:" or "https:" scheme, or an
128 explicit port should be provided.
129
130 ***********************************************************************/
131
132 this (RequestMethod method, char[] url)
133 {
134 this (method, new Uri(url));
135 }
136
137 /***********************************************************************
138
139 Create a client with the provided Uri instance. The Uri should
140 be fully qualified with an "http:" or "https:" scheme, or an
141 explicit port should be provided.
142
143 ***********************************************************************/
144
145 this (RequestMethod method, Uri uri)
146 {
147 this.uri = uri;
148 this.method = method;
149
150 responseLine = new ResponseLine;
151 headersIn = new HttpHeadersView;
152
153 tmp = new Buffer (1024 * 4);
154 paramsOut = new HttpParams (new Buffer (1024 * 4));
155 headersOut = new HttpHeaders (new Buffer (1024 * 4));
156 cookiesOut = new HttpCookies (headersOut);
157
158 // decode the host name (may take a second or two)
159 auto host = uri.getHost ();
160 if (host)
161 address = new InternetAddress (host, uri.getValidPort);
162 else
163 responseLine.error ("invalid url provided to HttpClient ctor");
164 }
165
166 /***********************************************************************
167
168 Get the current input headers, as returned by the host request.
169
170 ***********************************************************************/
171
172 HttpHeadersView getResponseHeaders()
173 {
174 return headersIn;
175 }
176
177 /***********************************************************************
178
179 Gain access to the request headers. Use this to add whatever
180 headers are required for a request.
181
182 ***********************************************************************/
183
184 HttpHeaders getRequestHeaders()
185 {
186 return headersOut;
187 }
188
189 /***********************************************************************
190
191 Gain access to the request parameters. Use this to add x=y
192 style parameters to the request. These will be appended to
193 the request assuming the original Uri does not contain any
194 of its own.
195
196 ***********************************************************************/
197
198 HttpParams getRequestParams()
199 {
200 return paramsOut;
201 }
202
203 /***********************************************************************
204
205 Return the Uri associated with this client
206
207 ***********************************************************************/
208
209 UriView getUri()
210 {
211 return uri;
212 }
213
214 /***********************************************************************
215
216 Return the response-line for the latest request. This takes
217 the form of "version status reason" as defined in the HTTP
218 RFC.
219
220 ***********************************************************************/
221
222 ResponseLine getResponse()
223 {
224 return responseLine;
225 }
226
227 /***********************************************************************
228
229 Return the HTTP status code set by the remote server
230
231 ***********************************************************************/
232
233 int getStatus()
234 {
235 return responseLine.getStatus;
236 }
237
238 /***********************************************************************
239
240 Return whether the response was OK or not
241
242 ***********************************************************************/
243
244 bool isResponseOK()
245 {
246 return getStatus is HttpResponseCode.OK;
247 }
248
249 /***********************************************************************
250
251 Add a cookie to the outgoing headers
252
253 ***********************************************************************/
254
255 void addCookie (Cookie cookie)
256 {
257 cookiesOut.add (cookie);
258 }
259
260 /***********************************************************************
261
262 Close all resources used by a request. You must invoke this
263 between successive open() calls.
264
265 ***********************************************************************/
266
267 void close ()
268 {
269 if (socket)
270 {
271 socket.shutdown;
272 socket.detach;
273 socket = null;
274 }
275 }
276
277 /***********************************************************************
278
279 Reset the client such that it is ready for a new request.
280
281 ***********************************************************************/
282
283 void reset ()
284 {
285 headersIn.reset;
286 headersOut.reset;
287 paramsOut.reset;
288 }
289
290 /***********************************************************************
291
292 enable/disable the internal redirection suppport
293
294 ***********************************************************************/
295
296 void enableRedirect (bool yes)
297 {
298 doRedirect = yes;
299 }
300
301 /***********************************************************************
302
303 set timeout period for read operation
304
305 ***********************************************************************/
306
307 void setTimeout (TimeSpan interval)
308 {
309 timeout = interval;
310 }
311
312 /**
313 * Deprecated: use setTimeout(TimeSpan) instead
314 */
315 deprecated void setTimeout(double interval)
316 {
317 setTimeout(TimeSpan.interval(interval));
318 }
319
320 /***********************************************************************
321
322 Overridable method to create a Socket. You may find a need
323 to override this for some purpose; perhaps to add input or
324 output filters.
325
326 ***********************************************************************/
327
328 protected SocketConduit createSocket ()
329 {
330 return new SocketConduit;
331 }
332
333 /***********************************************************************
334
335 Make a request for the resource specified via the constructor,
336 using the specified timeout period (in milli-seconds).The
337 return value represents the input buffer, from which all
338 returned headers and content may be accessed.
339
340 ***********************************************************************/
341
342 IBuffer open (IBuffer buffer = null)
343 {
344 return open (method, null, buffer);
345 }
346
347 /***********************************************************************
348
349 Make a request for the resource specified via the constructor,
350 using a callback for pumping additional data to the host. This
351 defaults to a three-second timeout period. The return value
352 represents the input buffer, from which all returned headers
353 and content may be accessed.
354
355 ***********************************************************************/
356
357 IBuffer open (Pump pump, IBuffer buffer = null)
358 {
359 return open (method, pump, buffer);
360 }
361
362 /***********************************************************************
363
364 Make a request for the resource specified via the constructor
365 using the specified timeout period (in micro-seconds), and a
366 user-defined callback for pumping additional data to the host.
367 The callback would be used when uploading data during a 'put'
368 operation (or equivalent). The return value represents the
369 input buffer, from which all returned headers and content may
370 be accessed.
371
372 Note that certain request-headers may generated automatically
373 if they are not present. These include a Host header and, in
374 the case of Post, both ContentType & ContentLength for a query
375 type of request. The latter two are *not* produced for Post
376 requests with 'pump' specified ~ when using 'pump' to output
377 additional content, you must explicitly set your own headers.
378
379 Note also that IOException instances may be thrown. These
380 should be caught by the client to ensure a close() operation
381 is always performed
382
383 ***********************************************************************/
384
385 private IBuffer open (RequestMethod method, Pump pump, IBuffer input)
386 {
387 // create socket, and connect it
388 socket = createSocket;
389 socket.setTimeout (timeout);
390 socket.connect (address);
391
392 // create buffers for input and output
393 if (input)
394 input.clear, input.setConduit (socket);
395 else
396 input = new Buffer (socket);
397 output = new Buffer (socket);
398
399 // save for read() method
400 this.input = input;
401
402 // setup a Host header
403 if (headersOut.get (HttpHeader.Host, null) is null)
404 headersOut.add (HttpHeader.Host, uri.getHost);
405
406 // http/1.0 needs connection:close
407 headersOut.add (HttpHeader.Connection, "close");
408
409 // attach/extend query parameters if user has added some
410 tmp.clear;
411 auto query = uri.extendQuery (paramsOut.formatTokens(tmp, "&"));
412
413 // patch request path?
414 auto path = uri.getPath;
415 if (path.length is 0)
416 path = "/";
417
418 // format encoded request
419 output (method.name) (" "), uri.encode (&output.consume, path, uri.IncPath);
420
421 // should we emit query as part of request line?
422 if (query.length)
423 if (method is Get)
424 output ("?"), uri.encode (&output.consume, query, uri.IncQueryAll);
425 else
426 if (method is Post && pump.funcptr is null)
427 {
428 // we're POSTing query text - add default info
429 if (headersOut.get (HttpHeader.ContentType, null) is null)
430 headersOut.add (HttpHeader.ContentType, "application/x-www-form-urlencoded");
431
432 if (headersOut.get (HttpHeader.ContentLength, null) is null)
433 headersOut.addInt (HttpHeader.ContentLength, query.length);
434 }
435
436 // complete the request line, and emit headers too
437 output (" ") (DefaultHttpVersion) (HttpConst.Eol);
438
439 headersOut.produce (&output.consume, HttpConst.Eol);
440 output (HttpConst.Eol);
441
442 // user has additional data to send?
443 if (pump.funcptr)
444 pump (output);
445 else
446 // send encoded POST query instead?
447 if (method is Post && query.length)
448 uri.encode (&output.consume, query, uri.IncQueryAll);
449
450 // send entire request
451 output.flush;
452
453 // Token for initial parsing of input header lines
454 auto line = new LineIterator!(char) (input);
455
456 // skip any blank lines
457 while (line.next && line.get.length is 0)
458 {}
459
460 // throw if we experienced a timeout
461 if (socket.hadTimeout)
462 responseLine.error ("response timeout");
463
464 // is this a bogus request?
465 if (line.get.length is 0)
466 responseLine.error ("truncated response");
467
468 // read response line
469 responseLine.parse (line.get);
470
471 // parse incoming headers
472 headersIn.reset.parse (input);
473
474 // check for redirection
475 if (doRedirect)
476 switch (responseLine.getStatus)
477 {
478 case HttpResponseCode.SeeOther:
479 case HttpResponseCode.MovedPermanently:
480 case HttpResponseCode.MovedTemporarily:
481 case HttpResponseCode.TemporaryRedirect:
482 // drop this connection
483 close;
484
485 // remove any existing Host header
486 headersOut.remove (HttpHeader.Host);
487
488 // parse redirected uri
489 auto redirect = headersIn.get (HttpHeader.Location, "[missing url]");
490 uri.relParse (redirect);
491
492 // decode the host name (may take a second or two)
493 auto host = uri.getHost();
494 if (host)
495 address = new InternetAddress (uri.getHost, uri.getValidPort);
496 else
497 responseLine.error ("redirect has invalid url: "~redirect);
498
499 // figure out what to do
500 if (method is Get || method is Head)
501 return open (method, pump, input);
502 else
503 if (method is Post)
504 return redirectPost (pump, input, responseLine.getStatus);
505 else
506 responseLine.error ("unexpected redirect for method "~method.name);
507 default:
508 break;
509 }
510
511 // return the input buffer
512 return input;
513 }
514
515 /***********************************************************************
516
517 Read the content from the returning input stream, up to a
518 maximum length, and pass content to the given sink delegate
519 as it arrives.
520
521 Exits when length bytes have been processed, or an Eof is
522 seen on the stream.
523
524 ***********************************************************************/
525
526 void read (void delegate (void[]) sink, long len = long.max)
527 {
528 while (len > 0)
529 {
530 auto content = input.slice;
531 if ((len -= content.length) < 0)
532 {
533 content = content [0 .. $ + cast(size_t) len];
534 sink (content);
535 input.skip (content.length);
536 }
537 else
538 {
539 sink (content);
540 input.clear;
541 if (input.fill(input.input) is socket.Eof)
542 break;
543 }
544 }
545 }
546
547 /***********************************************************************
548
549 Handle redirection of Post
550
551 Guidance for the default behaviour came from this page:
552 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html
553
554 ***********************************************************************/
555
556 IBuffer redirectPost (Pump pump, IBuffer input, int status)
557 {
558 switch (status)
559 {
560 // use Get method to complete the Post
561 case HttpResponseCode.SeeOther:
562 case HttpResponseCode.MovedTemporarily:
563
564 // remove POST headers first!
565 headersOut.remove (HttpHeader.ContentLength);
566 headersOut.remove (HttpHeader.ContentType);
567 paramsOut.reset;
568 return open (Get, null, input);
569
570 // try entire Post again, if user say OK
571 case HttpResponseCode.MovedPermanently:
572 case HttpResponseCode.TemporaryRedirect:
573 if (canRepost (status))
574 return open (this.method, pump, input);
575 // fall through!
576
577 default:
578 responseLine.error ("Illegal redirection of Post");
579 }
580 return null;
581 }
582
583 /***********************************************************************
584
585 Handle user-notification of Post redirection. This should
586 be overridden appropriately.
587
588 Guidance for the default behaviour came from this page:
589 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html
590
591 ***********************************************************************/
592
593 bool canRepost (uint status)
594 {
595 return false;
596 }
597 }
598
599
600 /******************************************************************************
601
602 Class to represent an HTTP response-line
603
604 ******************************************************************************/
605
606 private class ResponseLine : HttpTriplet
607 {
608 private char[] vers,
609 reason;
610 private int status;
611
612 /**********************************************************************
613
614 test the validity of these tokens
615
616 **********************************************************************/
617
618 void test ()
619 {
620 vers = tokens[0];
621 reason = tokens[2];
622 status = cast(int) Integer.convert (tokens[1]);
623 if (status is 0)
624 {
625 status = cast(int) Integer.convert (tokens[2]);
626 if (status is 0)
627 error ("Invalid HTTP response: '"~tokens[0]~"' '"~tokens[1]~"' '" ~tokens[2] ~"'");
628 }
629 }
630
631 /**********************************************************************
632
633 Return HTTP version
634
635 **********************************************************************/
636
637 char[] getVersion ()
638 {
639 return vers;
640 }
641
642 /**********************************************************************
643
644 Return reason text
645
646 **********************************************************************/
647
648 char[] getReason ()
649 {
650 return reason;
651 }
652
653 /**********************************************************************
654
655 Return status integer
656
657 **********************************************************************/
658
659 int getStatus ()
660 {
661 return status;
662 }
663 }