diff --git a/docs/docs/user-guide/cli-options.md b/docs/docs/user-guide/cli-options.md index 3f8c940cd..298b366eb 100644 --- a/docs/docs/user-guide/cli-options.md +++ b/docs/docs/user-guide/cli-options.md @@ -161,8 +161,6 @@ Options: econds) after behaviors before movin g on to next page [number] [default: 0] - --dedupPolicy Deduplication policy - [string] [choices: "skip", "revisit", "keep"] [default: "skip"] --profile Path or HTTP(S) URL to tar.gz file w hich contains the browser profile di rectory [string] @@ -202,7 +200,7 @@ Options: [number] [default: 0] --diskUtilization If set, save state and exit if disk utilization exceeds this percentage - value [number] [default: 90] + value [number] [default: 0] --timeLimit If set, save state and exit after ti me limit, in seconds [number] [default: 0] diff --git a/package.json b/package.json index e62a23d6e..0b219ed40 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "browsertrix-crawler", - "version": "1.6.2", + "version": "1.6.3", "main": "browsertrix-crawler", "type": "module", "repository": "https://github.com/webrecorder/browsertrix-crawler", @@ -18,7 +18,7 @@ "dependencies": { "@novnc/novnc": "1.4.0", "@puppeteer/replay": "^3.1.1", - "@webrecorder/wabac": "^2.22.16", + "@webrecorder/wabac": "^2.23.3", "browsertrix-behaviors": "^0.8.5", "client-zip": "^2.4.5", "css-selector-parser": "^3.0.5", diff --git a/src/util/argParser.ts b/src/util/argParser.ts index c0564ec9a..73da878ba 100644 --- a/src/util/argParser.ts +++ b/src/util/argParser.ts @@ -392,13 +392,6 @@ class ArgParser { type: "number", }, - dedupPolicy: { - describe: "Deduplication policy", - default: "skip", - type: "string", - choices: ["skip", "revisit", "keep"], - }, - profile: { describe: "Path or HTTP(S) URL to tar.gz file which contains the browser profile directory", @@ -475,7 +468,7 @@ class ArgParser { describe: "If set, save state and exit if disk utilization exceeds this percentage value", type: "number", - default: 90, + default: 0, }, timeLimit: { diff --git a/src/util/recorder.ts b/src/util/recorder.ts index 04c91be43..aaa4a425f 100644 --- a/src/util/recorder.ts +++ b/src/util/recorder.ts @@ -41,7 +41,7 @@ const WRITE_DUPE_KEY = "s:writedupe"; const MIME_EVENT_STREAM = "text/event-stream"; const RW_MIME_TYPES = [ - "application/x-mpegURL", + "application/x-mpegurl", "application/vnd.apple.mpegurl", "application/dash+xml", "text/html", @@ -1117,11 +1117,13 @@ export class Recorder extends EventEmitter { return false; } + contentType = contentType.toLowerCase(); + let newString = null; let string = null; switch (contentType) { - case "application/x-mpegURL": + case "application/x-mpegurl": case "application/vnd.apple.mpegurl": string = payload.toString(); newString = rewriteHLS(string, { save: extraOpts }); @@ -1171,7 +1173,7 @@ export class Recorder extends EventEmitter { return true; } - if (RW_MIME_TYPES.includes(contentType)) { + if (RW_MIME_TYPES.includes(contentType.toLowerCase())) { return true; } diff --git a/tsconfig.json b/tsconfig.json index 50ecc48c8..e913ef61c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -101,8 +101,8 @@ // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ /* Completeness */ - // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ - "skipLibCheck": false /* Skip type checking all .d.ts files. */ + "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ }, "include": ["src/**/*"] diff --git a/yarn.lock b/yarn.lock index 014c76917..4cb40b072 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1134,16 +1134,16 @@ resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== -"@webrecorder/wabac@^2.22.16": - version "2.22.16" - resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.22.16.tgz#8b9684569b373b8e930852bce4512e2bd2810d65" - integrity sha512-n39kwNOD/bKpAFwQ8AXImFqOUhfqUYoz41E0baGfoXydnJc2LKiS7SMqg3wDHazZH3y2DVlUpPknrD7UM75g0A== +"@webrecorder/wabac@^2.23.3": + version "2.23.3" + resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.23.3.tgz#405f53649183c54fd116e334eae2666d6514a341" + integrity sha512-NlPNGNmilNf/NEqHbCNPcib4GNnZKQJKK3PIiI0BvEdem/TEjvcn5wEBbUntTYn+VwrhX36QY2HC7Iag+dVnvw== dependencies: "@peculiar/asn1-ecc" "^2.3.4" "@peculiar/asn1-schema" "^2.3.3" "@peculiar/x509" "^1.9.2" "@types/js-levenshtein" "^1.1.3" - "@webrecorder/wombat" "^3.8.12" + "@webrecorder/wombat" "^3.8.13" acorn "^8.10.0" auto-js-ipfs "^2.1.1" base64-js "^1.5.1" @@ -1164,10 +1164,10 @@ stream-browserify "^3.0.0" warcio "^2.4.3" -"@webrecorder/wombat@^3.8.12": - version "3.8.12" - resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.12.tgz#8d44ce0b1bda57ae496f2cf44d4d967feaa019fd" - integrity sha512-4lbjp0XKPBRjyxEZBDZTPEGO6msBX8+irkRzbgFcmnxs5Ln4O9QbYDAr/RB4vTsFlytWcC68VBLwZx2SfIE8dw== +"@webrecorder/wombat@^3.8.13": + version "3.8.13" + resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.13.tgz#264f639dd102dca415f5d01a649d6b95dfac9779" + integrity sha512-gg80bEpJE+2Wn0ZTbfCkt9+vTftJemBwAWe9TYXo7ErCX1v7RbIrZ5LfkjSWx3vCx6R4V31DxXk1mycsVrEapA== dependencies: warcio "^2.4.0"