From a57b576962d48ced2ccddeb1a6540223f60ccc20 Mon Sep 17 00:00:00 2001 From: watsonb8 Date: Sat, 19 Dec 2020 17:09:39 -0500 Subject: [PATCH] Wip --- .env.schema | 3 +- package-lock.json | 295 ++++++++++++++++++++++++++++++++++++ package.json | 2 + scripts/detectFromImage.ts | 74 +++++++++ src/common.ts | 18 ++- src/config.ts | 1 + src/homeLocationPlatform.ts | 3 +- src/monitor/monitor.ts | 35 ++++- src/rtsp/rtsp.ts | 1 + src/trainer.ts | 32 ++-- 10 files changed, 443 insertions(+), 21 deletions(-) create mode 100644 scripts/detectFromImage.ts diff --git a/.env.schema b/.env.schema index 314354c..259576e 100644 --- a/.env.schema +++ b/.env.schema @@ -1,3 +1,4 @@ REF_IMAGE_DIR= TRAINED_MODEL_DIR= -OUT_DIR= \ No newline at end of file +OUT_DIR= +CONFIDENCE= \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 65b0804..08ce673 100644 --- a/package-lock.json +++ b/package-lock.json @@ -135,6 +135,14 @@ "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.27.tgz", "integrity": "sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE=" }, + "@types/sharp": { + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/@types/sharp/-/sharp-0.26.1.tgz", + "integrity": "sha512-vOFcnP0+aQFDb+ToKVIj8ZV6xQ7pNYGGPeYweLHxyjoQUcIGj8iY9R3OVmJyRR5KUkb0Y4obBbMjoTrBXw6AQA==", + "requires": { + "@types/node": "*" + } + }, "@types/webgl-ext": { "version": "0.0.30", "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", @@ -249,6 +257,33 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bl": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", + "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "bonjour-hap": { "version": "3.5.11", "resolved": "https://registry.npmjs.org/bonjour-hap/-/bonjour-hap-3.5.11.tgz", @@ -271,6 +306,15 @@ "concat-map": "0.0.1" } }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -350,6 +394,30 @@ "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" }, + "color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/color/-/color-3.1.3.tgz", + "integrity": "sha512-xgXAcTHa2HeFCGLE9Xs/R82hujGtu9Jd9x4NW3T34+OMs7VoPsjwzRczKHvTAHeJwWFwX5j15+MgAppE8ztObQ==", + "requires": { + "color-convert": "^1.9.1", + "color-string": "^1.5.4" + }, + "dependencies": { + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + } + } + }, "color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -363,6 +431,15 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "color-string": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.4.tgz", + "integrity": "sha512-57yF5yt8Xa3czSEW1jfQDE79Idk0+AkN/4KWad6tbdxUmAs3MvjxlWSWD4deYytcRfoZ9nhKyFl1kj5tBvidbw==", + "requires": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -611,6 +688,14 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, "es-abstract": { "version": "1.18.0-next.1", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.1.tgz", @@ -684,6 +769,11 @@ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, + "expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==" + }, "fast-srp-hap": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/fast-srp-hap/-/fast-srp-hap-2.0.2.tgz", @@ -716,6 +806,11 @@ "mime-types": "^2.1.12" } }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "fs-minipass": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz", @@ -813,6 +908,11 @@ "has-symbols": "^1.0.1" } }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=" + }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -922,6 +1022,11 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, "ignore-walk": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", @@ -961,6 +1066,11 @@ "integrity": "sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA==", "dev": true }, + "is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "is-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.0.tgz", @@ -1102,6 +1212,21 @@ "p-locate": "^4.1.0" } }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + }, + "dependencies": { + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, "make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -1164,6 +1289,11 @@ "minimist": "^1.2.5" } }, + "mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -1190,6 +1320,11 @@ "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==" }, + "napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, "needle": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/needle/-/needle-2.5.2.tgz", @@ -1200,6 +1335,19 @@ "sax": "^1.2.4" } }, + "node-abi": { + "version": "2.19.3", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.19.3.tgz", + "integrity": "sha512-9xZrlyfvKhWme2EXFKQhZRp1yNWT/uI1luYPr3sFl+H4keYY4xR+1jO7mvTTijIsHf1M+QDe9uWuKeEpLInIlg==", + "requires": { + "semver": "^5.4.1" + } + }, + "node-addon-api": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.0.2.tgz", + "integrity": "sha512-+D4s2HCnxPd5PjjI0STKwncjXTUKKqm74MDMz9OPXavjsGmjkvwgLtA5yoxJUdmpj52+2u+RrXgPipahKczMKg==" + }, "node-fetch": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", @@ -1268,6 +1416,11 @@ } } }, + "noop-logger": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", + "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=" + }, "noop6": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/noop6/-/noop6-1.0.9.tgz", @@ -1427,6 +1580,28 @@ "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" }, + "prebuild-install": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-6.0.0.tgz", + "integrity": "sha512-h2ZJ1PXHKWZpp1caLw0oX9sagVpL2YTk+ZwInQbQ3QqNd4J03O6MpFNmMTJlkfgPENWqe5kP0WjQLqz5OjLfsw==", + "requires": { + "detect-libc": "^1.0.3", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^2.7.0", + "noop-logger": "^0.1.1", + "npmlog": "^4.0.1", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^3.0.3", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0", + "which-pm-runs": "^1.0.0" + } + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -1437,6 +1612,15 @@ "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "q": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/q/-/q-1.1.2.tgz", @@ -1559,6 +1743,61 @@ "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" }, + "sharp": { + "version": "0.26.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.26.3.tgz", + "integrity": "sha512-NdEJ9S6AMr8Px0zgtFo1TJjMK/ROMU92MkDtYn2BBrDjIx3YfH9TUyGdzPC+I/L619GeYQc690Vbaxc5FPCCWg==", + "requires": { + "array-flatten": "^3.0.0", + "color": "^3.1.3", + "detect-libc": "^1.0.3", + "node-addon-api": "^3.0.2", + "npmlog": "^4.1.2", + "prebuild-install": "^6.0.0", + "semver": "^7.3.2", + "simple-get": "^4.0.0", + "tar-fs": "^2.1.1", + "tunnel-agent": "^0.6.0" + }, + "dependencies": { + "array-flatten": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-3.0.0.tgz", + "integrity": "sha512-zPMVc3ZYlGLNk4mpK1NzP2wg0ml9t7fUgDsayR5Y5rSzxQilzR9FGu/EH2jQOcKSAeAfWeylyW8juy3OkWRvNA==" + }, + "decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "requires": { + "mimic-response": "^3.1.0" + } + }, + "mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" + }, + "semver": { + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz", + "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==", + "requires": { + "lru-cache": "^6.0.0" + } + }, + "simple-get": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.0.tgz", + "integrity": "sha512-ZalZGexYr3TA0SwySsr5HlgOOinS4Jsa8YB2GJ6lUNAazyAu4KG/VmzMTwAt2YVXzzVj8QmefmAonZIK2BSGcQ==", + "requires": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + } + } + }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -1602,6 +1841,14 @@ "simple-concat": "^1.0.0" } }, + "simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", + "requires": { + "is-arrayish": "^0.3.1" + } + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -1696,6 +1943,41 @@ "yallist": "^3.0.3" } }, + "tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "tar-stream": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz", + "integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==", + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "through2": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", @@ -1738,6 +2020,14 @@ "tslib": "^1.9.3" } }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, "tweetnacl": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", @@ -1808,6 +2098,11 @@ "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", "dev": true }, + "which-pm-runs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", + "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=" + }, "which-typed-array": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.2.tgz", diff --git a/package.json b/package.json index 0bf7424..6d02aa3 100644 --- a/package.json +++ b/package.json @@ -27,10 +27,12 @@ "@tensorflow/tfjs": "^2.6.0", "@tensorflow/tfjs-node": "^2.6.0", "@types/mime-types": "^2.1.0", + "@types/sharp": "^0.26.1", "@vladmandic/face-api": "^0.8.8", "canvas": "^2.6.1", "dotenv-extended": "^2.9.0", "mime-types": "^2.1.27", + "sharp": "^0.26.3", "tsyringe": "^4.4.0" }, "devDependencies": { diff --git a/scripts/detectFromImage.ts b/scripts/detectFromImage.ts new file mode 100644 index 0000000..b7da021 --- /dev/null +++ b/scripts/detectFromImage.ts @@ -0,0 +1,74 @@ +import { nets } from "@vladmandic/face-api"; +import * as faceapi from "@vladmandic/face-api"; +import canvas from "canvas"; +import fs from "fs"; +import * as path from "path"; +import dotenv from "dotenv-extended"; +import { getFaceDetectorOptions } from "../src/common"; +import * as mime from "mime-types"; +require("@tensorflow/tfjs-node"); + +const { Canvas, Image, ImageData } = canvas; +//@ts-ignore +faceapi.env.monkeyPatch({ Canvas, Image, ImageData }); + +const main = async () => { + dotenv.load({ + silent: false, + errorOnMissing: true, + }); + + const modelDir = process.env.TRAINED_MODEL_DIR as string; + const faceDetectionNet = nets.ssdMobilenetv1; + + await faceDetectionNet.loadFromDisk(path.join(__dirname, "../weights")); + await nets.faceLandmark68Net.loadFromDisk(path.join(__dirname, "../weights")); + await nets.faceRecognitionNet.loadFromDisk( + path.join(__dirname, "../weights") + ); + + const raw = fs.readFileSync(path.join(modelDir, "data.json"), "utf-8"); + const content = JSON.parse(raw); + const matcher = faceapi.FaceMatcher.fromJSON(content); + + let dir = path.join(process.env.OUT_DIR as string); + const files = fs.readdirSync(dir); + await Promise.all( + files.map(async (file: string) => { + const mimeType = mime.contentType(path.extname(path.join(dir, file))); + if (!mimeType || !mimeType.startsWith("image")) { + return; + } + const imgData = fs.readFileSync(path.join(dir, file)); + const input = ((await canvas.loadImage(imgData)) as unknown) as ImageData; + const out = faceapi.createCanvasFromMedia(input); + const resultsQuery = await faceapi + .detectAllFaces(out, getFaceDetectorOptions(faceDetectionNet)) + .withFaceLandmarks() + .withFaceDescriptors(); + + if (resultsQuery.length > 0) { + for (const res of resultsQuery) { + const bestMatch = matcher.findBestMatch(res.descriptor); + console.log( + `Face Detected with ${ + res.detection.score * 100 + }% accuracy and a distance of ${bestMatch.distance}: ${ + bestMatch.label + } in file ${path.join(dir, file)}` + ); + } + } else { + console.log( + `No faces detected in file ${path.join( + process.env.REF_IMAGE_DIR as string, + "aline", + file + )}` + ); + } + }) + ); +}; + +main(); diff --git a/src/common.ts b/src/common.ts index af8accc..cebbb79 100644 --- a/src/common.ts +++ b/src/common.ts @@ -9,10 +9,22 @@ export const minConfidence = 0.4; export const inputSize = 416; export const scoreThreshold = 0.5; -export const getFaceDetectorOptions = (net: faceapi.NeuralNetwork) => { +export const getFaceDetectorOptions = ( + net: faceapi.NeuralNetwork, + options?: { + confidence?: number; + inputSize?: number; + scoreThreshold?: number; + } +) => { return net === faceapi.nets.ssdMobilenetv1 - ? new faceapi.SsdMobilenetv1Options({ minConfidence }) - : new faceapi.TinyFaceDetectorOptions({ inputSize, scoreThreshold }); + ? new faceapi.SsdMobilenetv1Options({ + minConfidence: options?.confidence ?? minConfidence, + }) + : new faceapi.TinyFaceDetectorOptions({ + inputSize: options?.inputSize ?? inputSize, + scoreThreshold: options?.scoreThreshold ?? scoreThreshold, + }); }; export const saveFile = async ( diff --git a/src/config.ts b/src/config.ts index b45e570..47fa266 100644 --- a/src/config.ts +++ b/src/config.ts @@ -12,6 +12,7 @@ export interface IConfig extends PlatformConfig { debug?: boolean; writeOutput?: boolean; rate?: number; + confidence?: number; } export interface IRoom { diff --git a/src/homeLocationPlatform.ts b/src/homeLocationPlatform.ts index 4839976..a8f1dc9 100644 --- a/src/homeLocationPlatform.ts +++ b/src/homeLocationPlatform.ts @@ -87,7 +87,8 @@ export class HomeLocationPlatform implements DynamicPlatformPlugin { if (this.config.trainOnStartup) { const trainer = new Trainer( this.config.refImageDirectory, - this.config.trainedModelDirectory + this.config.trainedModelDirectory, + this.config.confidence ); faceMatcher = await trainer.train(true); } else { diff --git a/src/monitor/monitor.ts b/src/monitor/monitor.ts index 078e5d0..c773a0f 100644 --- a/src/monitor/monitor.ts +++ b/src/monitor/monitor.ts @@ -16,6 +16,7 @@ import { Event } from "../events"; import { IConfig } from "../config"; import { MonitorState } from "./monitorState"; import { IStream } from "./stream"; +import sharp from "sharp"; const { Canvas, Image, ImageData } = canvas; const defaultWatchDog = 30000; @@ -143,20 +144,36 @@ export class Monitor { this._config.watchdogTimeout ?? 30000 ); + const regularizedImgData = await sharp(args.data) + .modulate({ brightness: 3 }) + .sharpen() + .toBuffer(); + //Detect faces in image - const input = ((await canvas.loadImage(args.data)) as unknown) as ImageData; + const input = ((await canvas.loadImage( + regularizedImgData + )) as unknown) as ImageData; const out = faceapi.createCanvasFromMedia(input); const resultsQuery = await faceapi - .detectAllFaces(out, getFaceDetectorOptions(this._faceDetectionNet)) + .detectAllFaces( + out, + getFaceDetectorOptions(this._faceDetectionNet, { + confidence: this._config.confidence, + }) + ) .withFaceLandmarks() .withFaceDescriptors(); //Write to output image if (this._config.writeOutput) { - await saveFile(this._config.outputDirectory, room + ".jpg", args.data); + await saveFile( + this._config.outputDirectory, + room + ".jpg", + regularizedImgData + ); } for (const res of resultsQuery) { - const bestMatch = this._matcher.matchDescriptor(res.descriptor); + const bestMatch = this._matcher.findBestMatch(res.descriptor); const old = this._state[bestMatch.label]; this._state[bestMatch.label] = room; this._stateChangedEvent.fire(this, { @@ -165,9 +182,13 @@ export class Monitor { label: bestMatch.label, }); - if (this._config.debug) { - this._logger.info(`Face Detected: ${bestMatch.label} in room ${room}`); - } + this._logger.info( + `Face Detected with ${ + res.detection.score * 100 + }% accuracy and a distance of ${bestMatch.distance}: ${ + bestMatch.label + } in room ${room}` + ); } }; diff --git a/src/rtsp/rtsp.ts b/src/rtsp/rtsp.ts index f47fe94..3b58e20 100644 --- a/src/rtsp/rtsp.ts +++ b/src/rtsp/rtsp.ts @@ -78,6 +78,7 @@ export class Rtsp { const argStrings = [ `-rtsp_transport tcp`, `-i ${this._connecteionString}`, + `-qscale:v 1`, `-r ${this._options.rate ?? 10}`, `-vf mpdecimate,setpts=N/FRAME_RATE/TB`, this._options.image diff --git a/src/trainer.ts b/src/trainer.ts index bab8a27..87c64be 100644 --- a/src/trainer.ts +++ b/src/trainer.ts @@ -12,7 +12,11 @@ const { Canvas, Image, ImageData } = canvas; faceapi.env.monkeyPatch({ Canvas, Image, ImageData }); export class Trainer { - constructor(private _refImageDir: string, private _trainedModelDir: string) {} + constructor( + private _refImageDir: string, + private _trainedModelDir: string, + private _confidence?: number + ) {} public async train(writeToDisk: boolean): Promise { const faceDetectionNet = faceapi.nets.ssdMobilenetv1; @@ -24,18 +28,25 @@ export class Trainer { path.join(__dirname, "../weights") ); - const options = getFaceDetectorOptions(faceDetectionNet); + const options = getFaceDetectorOptions(faceDetectionNet, { + confidence: this._confidence, + }); const dirs = fs.readdirSync(this._refImageDir); const refs = []; for (const dir of dirs) { const descriptor = new LabeledFaceDescriptors(dir, []); - await this.getLabeledFaceDescriptorFromDir( - path.join(this._refImageDir, dir), - descriptor, - options - ); + try { + await this.getLabeledFaceDescriptorFromDir( + path.join(this._refImageDir, dir), + descriptor, + options + ); + } catch (err) { + console.log(err); + } + if (descriptor) { refs.push(descriptor); } @@ -93,8 +104,11 @@ export class Trainer { labeldFaceDescriptors.descriptors.push(descriptor.descriptor); } catch (err) { - throw new Error( - "An error occurred loading image at path: " + path.join(dir, file) + console.log( + "An error occurred loading image at " + + path.join(dir, file) + + ": " + + err.message ); } })