index.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404
  1. "use strict";
  2. var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
  3. function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
  4. return new (P || (P = Promise))(function (resolve, reject) {
  5. function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
  6. function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
  7. function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
  8. step((generator = generator.apply(thisArg, _arguments || [])).next());
  9. });
  10. };
  11. import { decode as base64Decode, encode as base64Encode } from "@ethersproject/base64";
  12. import { hexlify, isBytesLike } from "@ethersproject/bytes";
  13. import { shallowCopy } from "@ethersproject/properties";
  14. import { toUtf8Bytes, toUtf8String } from "@ethersproject/strings";
  15. import { Logger } from "@ethersproject/logger";
  16. import { version } from "./_version";
  17. const logger = new Logger(version);
  18. import { getUrl } from "./geturl";
  19. function staller(duration) {
  20. return new Promise((resolve) => {
  21. setTimeout(resolve, duration);
  22. });
  23. }
  24. function bodyify(value, type) {
  25. if (value == null) {
  26. return null;
  27. }
  28. if (typeof (value) === "string") {
  29. return value;
  30. }
  31. if (isBytesLike(value)) {
  32. if (type && (type.split("/")[0] === "text" || type.split(";")[0].trim() === "application/json")) {
  33. try {
  34. return toUtf8String(value);
  35. }
  36. catch (error) { }
  37. ;
  38. }
  39. return hexlify(value);
  40. }
  41. return value;
  42. }
  43. function unpercent(value) {
  44. return toUtf8Bytes(value.replace(/%([0-9a-f][0-9a-f])/gi, (all, code) => {
  45. return String.fromCharCode(parseInt(code, 16));
  46. }));
  47. }
  48. // This API is still a work in progress; the future changes will likely be:
  49. // - ConnectionInfo => FetchDataRequest<T = any>
  50. // - FetchDataRequest.body? = string | Uint8Array | { contentType: string, data: string | Uint8Array }
  51. // - If string => text/plain, Uint8Array => application/octet-stream (if content-type unspecified)
  52. // - FetchDataRequest.processFunc = (body: Uint8Array, response: FetchDataResponse) => T
  53. // For this reason, it should be considered internal until the API is finalized
  54. export function _fetchData(connection, body, processFunc) {
  55. // How many times to retry in the event of a throttle
  56. const attemptLimit = (typeof (connection) === "object" && connection.throttleLimit != null) ? connection.throttleLimit : 12;
  57. logger.assertArgument((attemptLimit > 0 && (attemptLimit % 1) === 0), "invalid connection throttle limit", "connection.throttleLimit", attemptLimit);
  58. const throttleCallback = ((typeof (connection) === "object") ? connection.throttleCallback : null);
  59. const throttleSlotInterval = ((typeof (connection) === "object" && typeof (connection.throttleSlotInterval) === "number") ? connection.throttleSlotInterval : 100);
  60. logger.assertArgument((throttleSlotInterval > 0 && (throttleSlotInterval % 1) === 0), "invalid connection throttle slot interval", "connection.throttleSlotInterval", throttleSlotInterval);
  61. const errorPassThrough = ((typeof (connection) === "object") ? !!(connection.errorPassThrough) : false);
  62. const headers = {};
  63. let url = null;
  64. // @TODO: Allow ConnectionInfo to override some of these values
  65. const options = {
  66. method: "GET",
  67. };
  68. let allow304 = false;
  69. let timeout = 2 * 60 * 1000;
  70. if (typeof (connection) === "string") {
  71. url = connection;
  72. }
  73. else if (typeof (connection) === "object") {
  74. if (connection == null || connection.url == null) {
  75. logger.throwArgumentError("missing URL", "connection.url", connection);
  76. }
  77. url = connection.url;
  78. if (typeof (connection.timeout) === "number" && connection.timeout > 0) {
  79. timeout = connection.timeout;
  80. }
  81. if (connection.headers) {
  82. for (const key in connection.headers) {
  83. headers[key.toLowerCase()] = { key: key, value: String(connection.headers[key]) };
  84. if (["if-none-match", "if-modified-since"].indexOf(key.toLowerCase()) >= 0) {
  85. allow304 = true;
  86. }
  87. }
  88. }
  89. options.allowGzip = !!connection.allowGzip;
  90. if (connection.user != null && connection.password != null) {
  91. if (url.substring(0, 6) !== "https:" && connection.allowInsecureAuthentication !== true) {
  92. logger.throwError("basic authentication requires a secure https url", Logger.errors.INVALID_ARGUMENT, { argument: "url", url: url, user: connection.user, password: "[REDACTED]" });
  93. }
  94. const authorization = connection.user + ":" + connection.password;
  95. headers["authorization"] = {
  96. key: "Authorization",
  97. value: "Basic " + base64Encode(toUtf8Bytes(authorization))
  98. };
  99. }
  100. if (connection.skipFetchSetup != null) {
  101. options.skipFetchSetup = !!connection.skipFetchSetup;
  102. }
  103. if (connection.fetchOptions != null) {
  104. options.fetchOptions = shallowCopy(connection.fetchOptions);
  105. }
  106. }
  107. const reData = new RegExp("^data:([^;:]*)?(;base64)?,(.*)$", "i");
  108. const dataMatch = ((url) ? url.match(reData) : null);
  109. if (dataMatch) {
  110. try {
  111. const response = {
  112. statusCode: 200,
  113. statusMessage: "OK",
  114. headers: { "content-type": (dataMatch[1] || "text/plain") },
  115. body: (dataMatch[2] ? base64Decode(dataMatch[3]) : unpercent(dataMatch[3]))
  116. };
  117. let result = response.body;
  118. if (processFunc) {
  119. result = processFunc(response.body, response);
  120. }
  121. return Promise.resolve(result);
  122. }
  123. catch (error) {
  124. logger.throwError("processing response error", Logger.errors.SERVER_ERROR, {
  125. body: bodyify(dataMatch[1], dataMatch[2]),
  126. error: error,
  127. requestBody: null,
  128. requestMethod: "GET",
  129. url: url
  130. });
  131. }
  132. }
  133. if (body) {
  134. options.method = "POST";
  135. options.body = body;
  136. if (headers["content-type"] == null) {
  137. headers["content-type"] = { key: "Content-Type", value: "application/octet-stream" };
  138. }
  139. if (headers["content-length"] == null) {
  140. headers["content-length"] = { key: "Content-Length", value: String(body.length) };
  141. }
  142. }
  143. const flatHeaders = {};
  144. Object.keys(headers).forEach((key) => {
  145. const header = headers[key];
  146. flatHeaders[header.key] = header.value;
  147. });
  148. options.headers = flatHeaders;
  149. const runningTimeout = (function () {
  150. let timer = null;
  151. const promise = new Promise(function (resolve, reject) {
  152. if (timeout) {
  153. timer = setTimeout(() => {
  154. if (timer == null) {
  155. return;
  156. }
  157. timer = null;
  158. reject(logger.makeError("timeout", Logger.errors.TIMEOUT, {
  159. requestBody: bodyify(options.body, flatHeaders["content-type"]),
  160. requestMethod: options.method,
  161. timeout: timeout,
  162. url: url
  163. }));
  164. }, timeout);
  165. }
  166. });
  167. const cancel = function () {
  168. if (timer == null) {
  169. return;
  170. }
  171. clearTimeout(timer);
  172. timer = null;
  173. };
  174. return { promise, cancel };
  175. })();
  176. const runningFetch = (function () {
  177. return __awaiter(this, void 0, void 0, function* () {
  178. for (let attempt = 0; attempt < attemptLimit; attempt++) {
  179. let response = null;
  180. try {
  181. response = yield getUrl(url, options);
  182. if (attempt < attemptLimit) {
  183. if (response.statusCode === 301 || response.statusCode === 302) {
  184. // Redirection; for now we only support absolute locataions
  185. const location = response.headers.location || "";
  186. if (options.method === "GET" && location.match(/^https:/)) {
  187. url = response.headers.location;
  188. continue;
  189. }
  190. }
  191. else if (response.statusCode === 429) {
  192. // Exponential back-off throttling
  193. let tryAgain = true;
  194. if (throttleCallback) {
  195. tryAgain = yield throttleCallback(attempt, url);
  196. }
  197. if (tryAgain) {
  198. let stall = 0;
  199. const retryAfter = response.headers["retry-after"];
  200. if (typeof (retryAfter) === "string" && retryAfter.match(/^[1-9][0-9]*$/)) {
  201. stall = parseInt(retryAfter) * 1000;
  202. }
  203. else {
  204. stall = throttleSlotInterval * parseInt(String(Math.random() * Math.pow(2, attempt)));
  205. }
  206. //console.log("Stalling 429");
  207. yield staller(stall);
  208. continue;
  209. }
  210. }
  211. }
  212. }
  213. catch (error) {
  214. response = error.response;
  215. if (response == null) {
  216. runningTimeout.cancel();
  217. logger.throwError("missing response", Logger.errors.SERVER_ERROR, {
  218. requestBody: bodyify(options.body, flatHeaders["content-type"]),
  219. requestMethod: options.method,
  220. serverError: error,
  221. url: url
  222. });
  223. }
  224. }
  225. let body = response.body;
  226. if (allow304 && response.statusCode === 304) {
  227. body = null;
  228. }
  229. else if (!errorPassThrough && (response.statusCode < 200 || response.statusCode >= 300)) {
  230. runningTimeout.cancel();
  231. logger.throwError("bad response", Logger.errors.SERVER_ERROR, {
  232. status: response.statusCode,
  233. headers: response.headers,
  234. body: bodyify(body, ((response.headers) ? response.headers["content-type"] : null)),
  235. requestBody: bodyify(options.body, flatHeaders["content-type"]),
  236. requestMethod: options.method,
  237. url: url
  238. });
  239. }
  240. if (processFunc) {
  241. try {
  242. const result = yield processFunc(body, response);
  243. runningTimeout.cancel();
  244. return result;
  245. }
  246. catch (error) {
  247. // Allow the processFunc to trigger a throttle
  248. if (error.throttleRetry && attempt < attemptLimit) {
  249. let tryAgain = true;
  250. if (throttleCallback) {
  251. tryAgain = yield throttleCallback(attempt, url);
  252. }
  253. if (tryAgain) {
  254. const timeout = throttleSlotInterval * parseInt(String(Math.random() * Math.pow(2, attempt)));
  255. //console.log("Stalling callback");
  256. yield staller(timeout);
  257. continue;
  258. }
  259. }
  260. runningTimeout.cancel();
  261. logger.throwError("processing response error", Logger.errors.SERVER_ERROR, {
  262. body: bodyify(body, ((response.headers) ? response.headers["content-type"] : null)),
  263. error: error,
  264. requestBody: bodyify(options.body, flatHeaders["content-type"]),
  265. requestMethod: options.method,
  266. url: url
  267. });
  268. }
  269. }
  270. runningTimeout.cancel();
  271. // If we had a processFunc, it either returned a T or threw above.
  272. // The "body" is now a Uint8Array.
  273. return body;
  274. }
  275. return logger.throwError("failed response", Logger.errors.SERVER_ERROR, {
  276. requestBody: bodyify(options.body, flatHeaders["content-type"]),
  277. requestMethod: options.method,
  278. url: url
  279. });
  280. });
  281. })();
  282. return Promise.race([runningTimeout.promise, runningFetch]);
  283. }
  284. export function fetchJson(connection, json, processFunc) {
  285. let processJsonFunc = (value, response) => {
  286. let result = null;
  287. if (value != null) {
  288. try {
  289. result = JSON.parse(toUtf8String(value));
  290. }
  291. catch (error) {
  292. logger.throwError("invalid JSON", Logger.errors.SERVER_ERROR, {
  293. body: value,
  294. error: error
  295. });
  296. }
  297. }
  298. if (processFunc) {
  299. result = processFunc(result, response);
  300. }
  301. return result;
  302. };
  303. // If we have json to send, we must
  304. // - add content-type of application/json (unless already overridden)
  305. // - convert the json to bytes
  306. let body = null;
  307. if (json != null) {
  308. body = toUtf8Bytes(json);
  309. // Create a connection with the content-type set for JSON
  310. const updated = (typeof (connection) === "string") ? ({ url: connection }) : shallowCopy(connection);
  311. if (updated.headers) {
  312. const hasContentType = (Object.keys(updated.headers).filter((k) => (k.toLowerCase() === "content-type")).length) !== 0;
  313. if (!hasContentType) {
  314. updated.headers = shallowCopy(updated.headers);
  315. updated.headers["content-type"] = "application/json";
  316. }
  317. }
  318. else {
  319. updated.headers = { "content-type": "application/json" };
  320. }
  321. connection = updated;
  322. }
  323. return _fetchData(connection, body, processJsonFunc);
  324. }
  325. export function poll(func, options) {
  326. if (!options) {
  327. options = {};
  328. }
  329. options = shallowCopy(options);
  330. if (options.floor == null) {
  331. options.floor = 0;
  332. }
  333. if (options.ceiling == null) {
  334. options.ceiling = 10000;
  335. }
  336. if (options.interval == null) {
  337. options.interval = 250;
  338. }
  339. return new Promise(function (resolve, reject) {
  340. let timer = null;
  341. let done = false;
  342. // Returns true if cancel was successful. Unsuccessful cancel means we're already done.
  343. const cancel = () => {
  344. if (done) {
  345. return false;
  346. }
  347. done = true;
  348. if (timer) {
  349. clearTimeout(timer);
  350. }
  351. return true;
  352. };
  353. if (options.timeout) {
  354. timer = setTimeout(() => {
  355. if (cancel()) {
  356. reject(new Error("timeout"));
  357. }
  358. }, options.timeout);
  359. }
  360. const retryLimit = options.retryLimit;
  361. let attempt = 0;
  362. function check() {
  363. return func().then(function (result) {
  364. // If we have a result, or are allowed null then we're done
  365. if (result !== undefined) {
  366. if (cancel()) {
  367. resolve(result);
  368. }
  369. }
  370. else if (options.oncePoll) {
  371. options.oncePoll.once("poll", check);
  372. }
  373. else if (options.onceBlock) {
  374. options.onceBlock.once("block", check);
  375. // Otherwise, exponential back-off (up to 10s) our next request
  376. }
  377. else if (!done) {
  378. attempt++;
  379. if (attempt > retryLimit) {
  380. if (cancel()) {
  381. reject(new Error("retry limit reached"));
  382. }
  383. return;
  384. }
  385. let timeout = options.interval * parseInt(String(Math.random() * Math.pow(2, attempt)));
  386. if (timeout < options.floor) {
  387. timeout = options.floor;
  388. }
  389. if (timeout > options.ceiling) {
  390. timeout = options.ceiling;
  391. }
  392. setTimeout(check, timeout);
  393. }
  394. return null;
  395. }, function (error) {
  396. if (cancel()) {
  397. reject(error);
  398. }
  399. });
  400. }
  401. check();
  402. });
  403. }
  404. //# sourceMappingURL=index.js.map