1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Classes to encapsulate a single HTTP request.
16
17 The classes implement a command pattern, with every
18 object supporting an execute() method that does the
19 actuall HTTP request.
20 """
21 from __future__ import absolute_import
22 import six
23 from six.moves import http_client
24 from six.moves import range
25
26 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
27
28 from six import BytesIO, StringIO
29 from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote
30
31 import base64
32 import copy
33 import gzip
34 import httplib2
35 import json
36 import logging
37 import mimetypes
38 import os
39 import random
40 import socket
41 import sys
42 import time
43 import uuid
44
45
46 try:
47 import ssl
48 except ImportError:
49 _ssl_SSLError = object()
50 else:
51 _ssl_SSLError = ssl.SSLError
52
53 from email.generator import Generator
54 from email.mime.multipart import MIMEMultipart
55 from email.mime.nonmultipart import MIMENonMultipart
56 from email.parser import FeedParser
57
58
59
60 try:
61 from oauth2client import util
62 except ImportError:
63 from oauth2client import _helpers as util
64
65 from googleapiclient import mimeparse
66 from googleapiclient.errors import BatchError
67 from googleapiclient.errors import HttpError
68 from googleapiclient.errors import InvalidChunkSizeError
69 from googleapiclient.errors import ResumableUploadError
70 from googleapiclient.errors import UnexpectedBodyError
71 from googleapiclient.errors import UnexpectedMethodError
72 from googleapiclient.model import JsonModel
73
74
75 LOGGER = logging.getLogger(__name__)
76
77 DEFAULT_CHUNK_SIZE = 512*1024
78
79 MAX_URI_LENGTH = 2048
80
81 _TOO_MANY_REQUESTS = 429
85 """Determines whether a response should be retried.
86
87 Args:
88 resp_status: The response status received.
89 content: The response content body.
90
91 Returns:
92 True if the response should be retried, otherwise False.
93 """
94
95 if resp_status >= 500:
96 return True
97
98
99 if resp_status == _TOO_MANY_REQUESTS:
100 return True
101
102
103
104 if resp_status == six.moves.http_client.FORBIDDEN:
105
106 if not content:
107 return False
108
109
110 try:
111 data = json.loads(content.decode('utf-8'))
112 reason = data['error']['errors'][0]['reason']
113 except (UnicodeDecodeError, ValueError, KeyError):
114 LOGGER.warning('Invalid JSON content from response: %s', content)
115 return False
116
117 LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason)
118
119
120 if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ):
121 return True
122
123
124 return False
125
126
127 -def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args,
128 **kwargs):
129 """Retries an HTTP request multiple times while handling errors.
130
131 If after all retries the request still fails, last error is either returned as
132 return value (for HTTP 5xx errors) or thrown (for ssl.SSLError).
133
134 Args:
135 http: Http object to be used to execute request.
136 num_retries: Maximum number of retries.
137 req_type: Type of the request (used for logging retries).
138 sleep, rand: Functions to sleep for random time between retries.
139 uri: URI to be requested.
140 method: HTTP method to be used.
141 args, kwargs: Additional arguments passed to http.request.
142
143 Returns:
144 resp, content - Response from the http request (may be HTTP 5xx).
145 """
146 resp = None
147 content = None
148 for retry_num in range(num_retries + 1):
149 if retry_num > 0:
150
151 sleep_time = rand() * 2 ** retry_num
152 LOGGER.warning(
153 'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s',
154 sleep_time, retry_num, num_retries, req_type, method, uri,
155 resp.status if resp else exception)
156 sleep(sleep_time)
157
158 try:
159 exception = None
160 resp, content = http.request(uri, method, *args, **kwargs)
161
162 except _ssl_SSLError as ssl_error:
163 exception = ssl_error
164 except socket.error as socket_error:
165
166 if socket.errno.errorcode.get(socket_error.errno) not in (
167 'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED', ):
168 raise
169 exception = socket_error
170
171 if exception:
172 if retry_num == num_retries:
173 raise exception
174 else:
175 continue
176
177 if not _should_retry_response(resp.status, content):
178 break
179
180 return resp, content
181
208
234
377
502
569
598
690
693 """Truncated stream.
694
695 Takes a stream and presents a stream that is a slice of the original stream.
696 This is used when uploading media in chunks. In later versions of Python a
697 stream can be passed to httplib in place of the string of data to send. The
698 problem is that httplib just blindly reads to the end of the stream. This
699 wrapper presents a virtual stream that only reads to the end of the chunk.
700 """
701
702 - def __init__(self, stream, begin, chunksize):
703 """Constructor.
704
705 Args:
706 stream: (io.Base, file object), the stream to wrap.
707 begin: int, the seek position the chunk begins at.
708 chunksize: int, the size of the chunk.
709 """
710 self._stream = stream
711 self._begin = begin
712 self._chunksize = chunksize
713 self._stream.seek(begin)
714
715 - def read(self, n=-1):
716 """Read n bytes.
717
718 Args:
719 n, int, the number of bytes to read.
720
721 Returns:
722 A string of length 'n', or less if EOF is reached.
723 """
724
725 cur = self._stream.tell()
726 end = self._begin + self._chunksize
727 if n == -1 or cur + n > end:
728 n = end - cur
729 return self._stream.read(n)
730
733 """Encapsulates a single HTTP request."""
734
735 @util.positional(4)
736 - def __init__(self, http, postproc, uri,
737 method='GET',
738 body=None,
739 headers=None,
740 methodId=None,
741 resumable=None):
742 """Constructor for an HttpRequest.
743
744 Args:
745 http: httplib2.Http, the transport object to use to make a request
746 postproc: callable, called on the HTTP response and content to transform
747 it into a data object before returning, or raising an exception
748 on an error.
749 uri: string, the absolute URI to send the request to
750 method: string, the HTTP method to use
751 body: string, the request body of the HTTP request,
752 headers: dict, the HTTP request headers
753 methodId: string, a unique identifier for the API method being called.
754 resumable: MediaUpload, None if this is not a resumbale request.
755 """
756 self.uri = uri
757 self.method = method
758 self.body = body
759 self.headers = headers or {}
760 self.methodId = methodId
761 self.http = http
762 self.postproc = postproc
763 self.resumable = resumable
764 self.response_callbacks = []
765 self._in_error_state = False
766
767
768 major, minor, params = mimeparse.parse_mime_type(
769 self.headers.get('content-type', 'application/json'))
770
771
772 self.body_size = len(self.body or '')
773
774
775 self.resumable_uri = None
776
777
778 self.resumable_progress = 0
779
780
781 self._rand = random.random
782 self._sleep = time.sleep
783
784 @util.positional(1)
785 - def execute(self, http=None, num_retries=0):
786 """Execute the request.
787
788 Args:
789 http: httplib2.Http, an http object to be used in place of the
790 one the HttpRequest request object was constructed with.
791 num_retries: Integer, number of times to retry with randomized
792 exponential backoff. If all retries fail, the raised HttpError
793 represents the last request. If zero (default), we attempt the
794 request only once.
795
796 Returns:
797 A deserialized object model of the response body as determined
798 by the postproc.
799
800 Raises:
801 googleapiclient.errors.HttpError if the response was not a 2xx.
802 httplib2.HttpLib2Error if a transport error has occured.
803 """
804 if http is None:
805 http = self.http
806
807 if self.resumable:
808 body = None
809 while body is None:
810 _, body = self.next_chunk(http=http, num_retries=num_retries)
811 return body
812
813
814
815 if 'content-length' not in self.headers:
816 self.headers['content-length'] = str(self.body_size)
817
818 if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET':
819 self.method = 'POST'
820 self.headers['x-http-method-override'] = 'GET'
821 self.headers['content-type'] = 'application/x-www-form-urlencoded'
822 parsed = urlparse(self.uri)
823 self.uri = urlunparse(
824 (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None,
825 None)
826 )
827 self.body = parsed.query
828 self.headers['content-length'] = str(len(self.body))
829
830
831 resp, content = _retry_request(
832 http, num_retries, 'request', self._sleep, self._rand, str(self.uri),
833 method=str(self.method), body=self.body, headers=self.headers)
834
835 for callback in self.response_callbacks:
836 callback(resp)
837 if resp.status >= 300:
838 raise HttpError(resp, content, uri=self.uri)
839 return self.postproc(resp, content)
840
841 @util.positional(2)
843 """add_response_headers_callback
844
845 Args:
846 cb: Callback to be called on receiving the response headers, of signature:
847
848 def cb(resp):
849 # Where resp is an instance of httplib2.Response
850 """
851 self.response_callbacks.append(cb)
852
853 @util.positional(1)
855 """Execute the next step of a resumable upload.
856
857 Can only be used if the method being executed supports media uploads and
858 the MediaUpload object passed in was flagged as using resumable upload.
859
860 Example:
861
862 media = MediaFileUpload('cow.png', mimetype='image/png',
863 chunksize=1000, resumable=True)
864 request = farm.animals().insert(
865 id='cow',
866 name='cow.png',
867 media_body=media)
868
869 response = None
870 while response is None:
871 status, response = request.next_chunk()
872 if status:
873 print "Upload %d%% complete." % int(status.progress() * 100)
874
875
876 Args:
877 http: httplib2.Http, an http object to be used in place of the
878 one the HttpRequest request object was constructed with.
879 num_retries: Integer, number of times to retry with randomized
880 exponential backoff. If all retries fail, the raised HttpError
881 represents the last request. If zero (default), we attempt the
882 request only once.
883
884 Returns:
885 (status, body): (ResumableMediaStatus, object)
886 The body will be None until the resumable media is fully uploaded.
887
888 Raises:
889 googleapiclient.errors.HttpError if the response was not a 2xx.
890 httplib2.HttpLib2Error if a transport error has occured.
891 """
892 if http is None:
893 http = self.http
894
895 if self.resumable.size() is None:
896 size = '*'
897 else:
898 size = str(self.resumable.size())
899
900 if self.resumable_uri is None:
901 start_headers = copy.copy(self.headers)
902 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
903 if size != '*':
904 start_headers['X-Upload-Content-Length'] = size
905 start_headers['content-length'] = str(self.body_size)
906
907 resp, content = _retry_request(
908 http, num_retries, 'resumable URI request', self._sleep, self._rand,
909 self.uri, method=self.method, body=self.body, headers=start_headers)
910
911 if resp.status == 200 and 'location' in resp:
912 self.resumable_uri = resp['location']
913 else:
914 raise ResumableUploadError(resp, content)
915 elif self._in_error_state:
916
917
918
919 headers = {
920 'Content-Range': 'bytes */%s' % size,
921 'content-length': '0'
922 }
923 resp, content = http.request(self.resumable_uri, 'PUT',
924 headers=headers)
925 status, body = self._process_response(resp, content)
926 if body:
927
928 return (status, body)
929
930 if self.resumable.has_stream():
931 data = self.resumable.stream()
932 if self.resumable.chunksize() == -1:
933 data.seek(self.resumable_progress)
934 chunk_end = self.resumable.size() - self.resumable_progress - 1
935 else:
936
937 data = _StreamSlice(data, self.resumable_progress,
938 self.resumable.chunksize())
939 chunk_end = min(
940 self.resumable_progress + self.resumable.chunksize() - 1,
941 self.resumable.size() - 1)
942 else:
943 data = self.resumable.getbytes(
944 self.resumable_progress, self.resumable.chunksize())
945
946
947 if len(data) < self.resumable.chunksize():
948 size = str(self.resumable_progress + len(data))
949
950 chunk_end = self.resumable_progress + len(data) - 1
951
952 headers = {
953 'Content-Range': 'bytes %d-%d/%s' % (
954 self.resumable_progress, chunk_end, size),
955
956
957 'Content-Length': str(chunk_end - self.resumable_progress + 1)
958 }
959
960 for retry_num in range(num_retries + 1):
961 if retry_num > 0:
962 self._sleep(self._rand() * 2**retry_num)
963 LOGGER.warning(
964 'Retry #%d for media upload: %s %s, following status: %d'
965 % (retry_num, self.method, self.uri, resp.status))
966
967 try:
968 resp, content = http.request(self.resumable_uri, method='PUT',
969 body=data,
970 headers=headers)
971 except:
972 self._in_error_state = True
973 raise
974 if not _should_retry_response(resp.status, content):
975 break
976
977 return self._process_response(resp, content)
978
980 """Process the response from a single chunk upload.
981
982 Args:
983 resp: httplib2.Response, the response object.
984 content: string, the content of the response.
985
986 Returns:
987 (status, body): (ResumableMediaStatus, object)
988 The body will be None until the resumable media is fully uploaded.
989
990 Raises:
991 googleapiclient.errors.HttpError if the response was not a 2xx or a 308.
992 """
993 if resp.status in [200, 201]:
994 self._in_error_state = False
995 return None, self.postproc(resp, content)
996 elif resp.status == 308:
997 self._in_error_state = False
998
999 try:
1000 self.resumable_progress = int(resp['range'].split('-')[1]) + 1
1001 except KeyError:
1002
1003 self.resumable_progress = 0
1004 if 'location' in resp:
1005 self.resumable_uri = resp['location']
1006 else:
1007 self._in_error_state = True
1008 raise HttpError(resp, content, uri=self.uri)
1009
1010 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
1011 None)
1012
1014 """Returns a JSON representation of the HttpRequest."""
1015 d = copy.copy(self.__dict__)
1016 if d['resumable'] is not None:
1017 d['resumable'] = self.resumable.to_json()
1018 del d['http']
1019 del d['postproc']
1020 del d['_sleep']
1021 del d['_rand']
1022
1023 return json.dumps(d)
1024
1025 @staticmethod
1027 """Returns an HttpRequest populated with info from a JSON object."""
1028 d = json.loads(s)
1029 if d['resumable'] is not None:
1030 d['resumable'] = MediaUpload.new_from_json(d['resumable'])
1031 return HttpRequest(
1032 http,
1033 postproc,
1034 uri=d['uri'],
1035 method=d['method'],
1036 body=d['body'],
1037 headers=d['headers'],
1038 methodId=d['methodId'],
1039 resumable=d['resumable'])
1040
1043 """Batches multiple HttpRequest objects into a single HTTP request.
1044
1045 Example:
1046 from googleapiclient.http import BatchHttpRequest
1047
1048 def list_animals(request_id, response, exception):
1049 \"\"\"Do something with the animals list response.\"\"\"
1050 if exception is not None:
1051 # Do something with the exception.
1052 pass
1053 else:
1054 # Do something with the response.
1055 pass
1056
1057 def list_farmers(request_id, response, exception):
1058 \"\"\"Do something with the farmers list response.\"\"\"
1059 if exception is not None:
1060 # Do something with the exception.
1061 pass
1062 else:
1063 # Do something with the response.
1064 pass
1065
1066 service = build('farm', 'v2')
1067
1068 batch = BatchHttpRequest()
1069
1070 batch.add(service.animals().list(), list_animals)
1071 batch.add(service.farmers().list(), list_farmers)
1072 batch.execute(http=http)
1073 """
1074
1075 @util.positional(1)
1076 - def __init__(self, callback=None, batch_uri=None):
1077 """Constructor for a BatchHttpRequest.
1078
1079 Args:
1080 callback: callable, A callback to be called for each response, of the
1081 form callback(id, response, exception). The first parameter is the
1082 request id, and the second is the deserialized response object. The
1083 third is an googleapiclient.errors.HttpError exception object if an HTTP error
1084 occurred while processing the request, or None if no error occurred.
1085 batch_uri: string, URI to send batch requests to.
1086 """
1087 if batch_uri is None:
1088 batch_uri = 'https://www.googleapis.com/batch'
1089 self._batch_uri = batch_uri
1090
1091
1092 self._callback = callback
1093
1094
1095 self._requests = {}
1096
1097
1098 self._callbacks = {}
1099
1100
1101 self._order = []
1102
1103
1104 self._last_auto_id = 0
1105
1106
1107 self._base_id = None
1108
1109
1110 self._responses = {}
1111
1112
1113 self._refreshed_credentials = {}
1114
1116 """Refresh the credentials and apply to the request.
1117
1118 Args:
1119 request: HttpRequest, the request.
1120 http: httplib2.Http, the global http object for the batch.
1121 """
1122
1123
1124
1125 creds = None
1126 if request.http is not None and hasattr(request.http.request,
1127 'credentials'):
1128 creds = request.http.request.credentials
1129 elif http is not None and hasattr(http.request, 'credentials'):
1130 creds = http.request.credentials
1131 if creds is not None:
1132 if id(creds) not in self._refreshed_credentials:
1133 creds.refresh(http)
1134 self._refreshed_credentials[id(creds)] = 1
1135
1136
1137
1138 if request.http is None or not hasattr(request.http.request,
1139 'credentials'):
1140 creds.apply(request.headers)
1141
1143 """Convert an id to a Content-ID header value.
1144
1145 Args:
1146 id_: string, identifier of individual request.
1147
1148 Returns:
1149 A Content-ID header with the id_ encoded into it. A UUID is prepended to
1150 the value because Content-ID headers are supposed to be universally
1151 unique.
1152 """
1153 if self._base_id is None:
1154 self._base_id = uuid.uuid4()
1155
1156 return '<%s+%s>' % (self._base_id, quote(id_))
1157
1159 """Convert a Content-ID header value to an id.
1160
1161 Presumes the Content-ID header conforms to the format that _id_to_header()
1162 returns.
1163
1164 Args:
1165 header: string, Content-ID header value.
1166
1167 Returns:
1168 The extracted id value.
1169
1170 Raises:
1171 BatchError if the header is not in the expected format.
1172 """
1173 if header[0] != '<' or header[-1] != '>':
1174 raise BatchError("Invalid value for Content-ID: %s" % header)
1175 if '+' not in header:
1176 raise BatchError("Invalid value for Content-ID: %s" % header)
1177 base, id_ = header[1:-1].rsplit('+', 1)
1178
1179 return unquote(id_)
1180
1182 """Convert an HttpRequest object into a string.
1183
1184 Args:
1185 request: HttpRequest, the request to serialize.
1186
1187 Returns:
1188 The request as a string in application/http format.
1189 """
1190
1191 parsed = urlparse(request.uri)
1192 request_line = urlunparse(
1193 ('', '', parsed.path, parsed.params, parsed.query, '')
1194 )
1195 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
1196 major, minor = request.headers.get('content-type', 'application/json').split('/')
1197 msg = MIMENonMultipart(major, minor)
1198 headers = request.headers.copy()
1199
1200 if request.http is not None and hasattr(request.http.request,
1201 'credentials'):
1202 request.http.request.credentials.apply(headers)
1203
1204
1205 if 'content-type' in headers:
1206 del headers['content-type']
1207
1208 for key, value in six.iteritems(headers):
1209 msg[key] = value
1210 msg['Host'] = parsed.netloc
1211 msg.set_unixfrom(None)
1212
1213 if request.body is not None:
1214 msg.set_payload(request.body)
1215 msg['content-length'] = str(len(request.body))
1216
1217
1218 fp = StringIO()
1219
1220 g = Generator(fp, maxheaderlen=0)
1221 g.flatten(msg, unixfrom=False)
1222 body = fp.getvalue()
1223
1224 return status_line + body
1225
1227 """Convert string into httplib2 response and content.
1228
1229 Args:
1230 payload: string, headers and body as a string.
1231
1232 Returns:
1233 A pair (resp, content), such as would be returned from httplib2.request.
1234 """
1235
1236 status_line, payload = payload.split('\n', 1)
1237 protocol, status, reason = status_line.split(' ', 2)
1238
1239
1240 parser = FeedParser()
1241 parser.feed(payload)
1242 msg = parser.close()
1243 msg['status'] = status
1244
1245
1246 resp = httplib2.Response(msg)
1247 resp.reason = reason
1248 resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
1249
1250 content = payload.split('\r\n\r\n', 1)[1]
1251
1252 return resp, content
1253
1255 """Create a new id.
1256
1257 Auto incrementing number that avoids conflicts with ids already used.
1258
1259 Returns:
1260 string, a new unique id.
1261 """
1262 self._last_auto_id += 1
1263 while str(self._last_auto_id) in self._requests:
1264 self._last_auto_id += 1
1265 return str(self._last_auto_id)
1266
1267 @util.positional(2)
1268 - def add(self, request, callback=None, request_id=None):
1269 """Add a new request.
1270
1271 Every callback added will be paired with a unique id, the request_id. That
1272 unique id will be passed back to the callback when the response comes back
1273 from the server. The default behavior is to have the library generate it's
1274 own unique id. If the caller passes in a request_id then they must ensure
1275 uniqueness for each request_id, and if they are not an exception is
1276 raised. Callers should either supply all request_ids or nevery supply a
1277 request id, to avoid such an error.
1278
1279 Args:
1280 request: HttpRequest, Request to add to the batch.
1281 callback: callable, A callback to be called for this response, of the
1282 form callback(id, response, exception). The first parameter is the
1283 request id, and the second is the deserialized response object. The
1284 third is an googleapiclient.errors.HttpError exception object if an HTTP error
1285 occurred while processing the request, or None if no errors occurred.
1286 request_id: string, A unique id for the request. The id will be passed to
1287 the callback with the response.
1288
1289 Returns:
1290 None
1291
1292 Raises:
1293 BatchError if a media request is added to a batch.
1294 KeyError is the request_id is not unique.
1295 """
1296 if request_id is None:
1297 request_id = self._new_id()
1298 if request.resumable is not None:
1299 raise BatchError("Media requests cannot be used in a batch request.")
1300 if request_id in self._requests:
1301 raise KeyError("A request with this ID already exists: %s" % request_id)
1302 self._requests[request_id] = request
1303 self._callbacks[request_id] = callback
1304 self._order.append(request_id)
1305
1306 - def _execute(self, http, order, requests):
1307 """Serialize batch request, send to server, process response.
1308
1309 Args:
1310 http: httplib2.Http, an http object to be used to make the request with.
1311 order: list, list of request ids in the order they were added to the
1312 batch.
1313 request: list, list of request objects to send.
1314
1315 Raises:
1316 httplib2.HttpLib2Error if a transport error has occured.
1317 googleapiclient.errors.BatchError if the response is the wrong format.
1318 """
1319 message = MIMEMultipart('mixed')
1320
1321 setattr(message, '_write_headers', lambda self: None)
1322
1323
1324 for request_id in order:
1325 request = requests[request_id]
1326
1327 msg = MIMENonMultipart('application', 'http')
1328 msg['Content-Transfer-Encoding'] = 'binary'
1329 msg['Content-ID'] = self._id_to_header(request_id)
1330
1331 body = self._serialize_request(request)
1332 msg.set_payload(body)
1333 message.attach(msg)
1334
1335
1336
1337 fp = StringIO()
1338 g = Generator(fp, mangle_from_=False)
1339 g.flatten(message, unixfrom=False)
1340 body = fp.getvalue()
1341
1342 headers = {}
1343 headers['content-type'] = ('multipart/mixed; '
1344 'boundary="%s"') % message.get_boundary()
1345
1346 resp, content = http.request(self._batch_uri, method='POST', body=body,
1347 headers=headers)
1348
1349 if resp.status >= 300:
1350 raise HttpError(resp, content, uri=self._batch_uri)
1351
1352
1353 header = 'content-type: %s\r\n\r\n' % resp['content-type']
1354
1355
1356 if six.PY3:
1357 content = content.decode('utf-8')
1358 for_parser = header + content
1359
1360 parser = FeedParser()
1361 parser.feed(for_parser)
1362 mime_response = parser.close()
1363
1364 if not mime_response.is_multipart():
1365 raise BatchError("Response not in multipart/mixed format.", resp=resp,
1366 content=content)
1367
1368 for part in mime_response.get_payload():
1369 request_id = self._header_to_id(part['Content-ID'])
1370 response, content = self._deserialize_response(part.get_payload())
1371
1372 if isinstance(content, six.text_type):
1373 content = content.encode('utf-8')
1374 self._responses[request_id] = (response, content)
1375
1376 @util.positional(1)
1378 """Execute all the requests as a single batched HTTP request.
1379
1380 Args:
1381 http: httplib2.Http, an http object to be used in place of the one the
1382 HttpRequest request object was constructed with. If one isn't supplied
1383 then use a http object from the requests in this batch.
1384
1385 Returns:
1386 None
1387
1388 Raises:
1389 httplib2.HttpLib2Error if a transport error has occured.
1390 googleapiclient.errors.BatchError if the response is the wrong format.
1391 """
1392
1393 if len(self._order) == 0:
1394 return None
1395
1396
1397 if http is None:
1398 for request_id in self._order:
1399 request = self._requests[request_id]
1400 if request is not None:
1401 http = request.http
1402 break
1403
1404 if http is None:
1405 raise ValueError("Missing a valid http object.")
1406
1407
1408
1409 if getattr(http.request, 'credentials', None) is not None:
1410 creds = http.request.credentials
1411 if not getattr(creds, 'access_token', None):
1412 LOGGER.info('Attempting refresh to obtain initial access_token')
1413 creds.refresh(http)
1414
1415 self._execute(http, self._order, self._requests)
1416
1417
1418
1419 redo_requests = {}
1420 redo_order = []
1421
1422 for request_id in self._order:
1423 resp, content = self._responses[request_id]
1424 if resp['status'] == '401':
1425 redo_order.append(request_id)
1426 request = self._requests[request_id]
1427 self._refresh_and_apply_credentials(request, http)
1428 redo_requests[request_id] = request
1429
1430 if redo_requests:
1431 self._execute(http, redo_order, redo_requests)
1432
1433
1434
1435
1436
1437 for request_id in self._order:
1438 resp, content = self._responses[request_id]
1439
1440 request = self._requests[request_id]
1441 callback = self._callbacks[request_id]
1442
1443 response = None
1444 exception = None
1445 try:
1446 if resp.status >= 300:
1447 raise HttpError(resp, content, uri=request.uri)
1448 response = request.postproc(resp, content)
1449 except HttpError as e:
1450 exception = e
1451
1452 if callback is not None:
1453 callback(request_id, response, exception)
1454 if self._callback is not None:
1455 self._callback(request_id, response, exception)
1456
1459 """Mock of HttpRequest.
1460
1461 Do not construct directly, instead use RequestMockBuilder.
1462 """
1463
1464 - def __init__(self, resp, content, postproc):
1465 """Constructor for HttpRequestMock
1466
1467 Args:
1468 resp: httplib2.Response, the response to emulate coming from the request
1469 content: string, the response body
1470 postproc: callable, the post processing function usually supplied by
1471 the model class. See model.JsonModel.response() as an example.
1472 """
1473 self.resp = resp
1474 self.content = content
1475 self.postproc = postproc
1476 if resp is None:
1477 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
1478 if 'reason' in self.resp:
1479 self.resp.reason = self.resp['reason']
1480
1482 """Execute the request.
1483
1484 Same behavior as HttpRequest.execute(), but the response is
1485 mocked and not really from an HTTP request/response.
1486 """
1487 return self.postproc(self.resp, self.content)
1488
1491 """A simple mock of HttpRequest
1492
1493 Pass in a dictionary to the constructor that maps request methodIds to
1494 tuples of (httplib2.Response, content, opt_expected_body) that should be
1495 returned when that method is called. None may also be passed in for the
1496 httplib2.Response, in which case a 200 OK response will be generated.
1497 If an opt_expected_body (str or dict) is provided, it will be compared to
1498 the body and UnexpectedBodyError will be raised on inequality.
1499
1500 Example:
1501 response = '{"data": {"id": "tag:google.c...'
1502 requestBuilder = RequestMockBuilder(
1503 {
1504 'plus.activities.get': (None, response),
1505 }
1506 )
1507 googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
1508
1509 Methods that you do not supply a response for will return a
1510 200 OK with an empty string as the response content or raise an excpetion
1511 if check_unexpected is set to True. The methodId is taken from the rpcName
1512 in the discovery document.
1513
1514 For more details see the project wiki.
1515 """
1516
1517 - def __init__(self, responses, check_unexpected=False):
1518 """Constructor for RequestMockBuilder
1519
1520 The constructed object should be a callable object
1521 that can replace the class HttpResponse.
1522
1523 responses - A dictionary that maps methodIds into tuples
1524 of (httplib2.Response, content). The methodId
1525 comes from the 'rpcName' field in the discovery
1526 document.
1527 check_unexpected - A boolean setting whether or not UnexpectedMethodError
1528 should be raised on unsupplied method.
1529 """
1530 self.responses = responses
1531 self.check_unexpected = check_unexpected
1532
1533 - def __call__(self, http, postproc, uri, method='GET', body=None,
1534 headers=None, methodId=None, resumable=None):
1535 """Implements the callable interface that discovery.build() expects
1536 of requestBuilder, which is to build an object compatible with
1537 HttpRequest.execute(). See that method for the description of the
1538 parameters and the expected response.
1539 """
1540 if methodId in self.responses:
1541 response = self.responses[methodId]
1542 resp, content = response[:2]
1543 if len(response) > 2:
1544
1545 expected_body = response[2]
1546 if bool(expected_body) != bool(body):
1547
1548
1549 raise UnexpectedBodyError(expected_body, body)
1550 if isinstance(expected_body, str):
1551 expected_body = json.loads(expected_body)
1552 body = json.loads(body)
1553 if body != expected_body:
1554 raise UnexpectedBodyError(expected_body, body)
1555 return HttpRequestMock(resp, content, postproc)
1556 elif self.check_unexpected:
1557 raise UnexpectedMethodError(methodId=methodId)
1558 else:
1559 model = JsonModel(False)
1560 return HttpRequestMock(None, '{}', model.response)
1561
1564 """Mock of httplib2.Http"""
1565
1566 - def __init__(self, filename=None, headers=None):
1567 """
1568 Args:
1569 filename: string, absolute filename to read response from
1570 headers: dict, header to return with response
1571 """
1572 if headers is None:
1573 headers = {'status': '200'}
1574 if filename:
1575 f = open(filename, 'rb')
1576 self.data = f.read()
1577 f.close()
1578 else:
1579 self.data = None
1580 self.response_headers = headers
1581 self.headers = None
1582 self.uri = None
1583 self.method = None
1584 self.body = None
1585 self.headers = None
1586
1587
1588 - def request(self, uri,
1589 method='GET',
1590 body=None,
1591 headers=None,
1592 redirections=1,
1593 connection_type=None):
1594 self.uri = uri
1595 self.method = method
1596 self.body = body
1597 self.headers = headers
1598 return httplib2.Response(self.response_headers), self.data
1599
1602 """Mock of httplib2.Http
1603
1604 Mocks a sequence of calls to request returning different responses for each
1605 call. Create an instance initialized with the desired response headers
1606 and content and then use as if an httplib2.Http instance.
1607
1608 http = HttpMockSequence([
1609 ({'status': '401'}, ''),
1610 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
1611 ({'status': '200'}, 'echo_request_headers'),
1612 ])
1613 resp, content = http.request("http://examples.com")
1614
1615 There are special values you can pass in for content to trigger
1616 behavours that are helpful in testing.
1617
1618 'echo_request_headers' means return the request headers in the response body
1619 'echo_request_headers_as_json' means return the request headers in
1620 the response body
1621 'echo_request_body' means return the request body in the response body
1622 'echo_request_uri' means return the request uri in the response body
1623 """
1624
1626 """
1627 Args:
1628 iterable: iterable, a sequence of pairs of (headers, body)
1629 """
1630 self._iterable = iterable
1631 self.follow_redirects = True
1632
1633 - def request(self, uri,
1634 method='GET',
1635 body=None,
1636 headers=None,
1637 redirections=1,
1638 connection_type=None):
1639 resp, content = self._iterable.pop(0)
1640 if content == 'echo_request_headers':
1641 content = headers
1642 elif content == 'echo_request_headers_as_json':
1643 content = json.dumps(headers)
1644 elif content == 'echo_request_body':
1645 if hasattr(body, 'read'):
1646 content = body.read()
1647 else:
1648 content = body
1649 elif content == 'echo_request_uri':
1650 content = uri
1651 if isinstance(content, six.text_type):
1652 content = content.encode('utf-8')
1653 return httplib2.Response(resp), content
1654
1657 """Set the user-agent on every request.
1658
1659 Args:
1660 http - An instance of httplib2.Http
1661 or something that acts like it.
1662 user_agent: string, the value for the user-agent header.
1663
1664 Returns:
1665 A modified instance of http that was passed in.
1666
1667 Example:
1668
1669 h = httplib2.Http()
1670 h = set_user_agent(h, "my-app-name/6.0")
1671
1672 Most of the time the user-agent will be set doing auth, this is for the rare
1673 cases where you are accessing an unauthenticated endpoint.
1674 """
1675 request_orig = http.request
1676
1677
1678 def new_request(uri, method='GET', body=None, headers=None,
1679 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
1680 connection_type=None):
1681 """Modify the request headers to add the user-agent."""
1682 if headers is None:
1683 headers = {}
1684 if 'user-agent' in headers:
1685 headers['user-agent'] = user_agent + ' ' + headers['user-agent']
1686 else:
1687 headers['user-agent'] = user_agent
1688 resp, content = request_orig(uri, method, body, headers,
1689 redirections, connection_type)
1690 return resp, content
1691
1692 http.request = new_request
1693 return http
1694
1697 """Tunnel PATCH requests over POST.
1698 Args:
1699 http - An instance of httplib2.Http
1700 or something that acts like it.
1701
1702 Returns:
1703 A modified instance of http that was passed in.
1704
1705 Example:
1706
1707 h = httplib2.Http()
1708 h = tunnel_patch(h, "my-app-name/6.0")
1709
1710 Useful if you are running on a platform that doesn't support PATCH.
1711 Apply this last if you are using OAuth 1.0, as changing the method
1712 will result in a different signature.
1713 """
1714 request_orig = http.request
1715
1716
1717 def new_request(uri, method='GET', body=None, headers=None,
1718 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
1719 connection_type=None):
1720 """Modify the request headers to add the user-agent."""
1721 if headers is None:
1722 headers = {}
1723 if method == 'PATCH':
1724 if 'oauth_token' in headers.get('authorization', ''):
1725 LOGGER.warning(
1726 'OAuth 1.0 request made with Credentials after tunnel_patch.')
1727 headers['x-http-method-override'] = "PATCH"
1728 method = 'POST'
1729 resp, content = request_orig(uri, method, body, headers,
1730 redirections, connection_type)
1731 return resp, content
1732
1733 http.request = new_request
1734 return http
1735