import { API, DynamicPlatformPlugin, Logger, PlatformAccessory, PlatformConfig, Service, Characteristic, } from "homebridge"; import { IConfig, isConfig } from "./config"; import * as faceapi from "@vladmandic/face-api"; import canvas from "canvas"; import fs, { lstatSync } from "fs"; import * as path from "path"; import { nets } from "@vladmandic/face-api"; import { LabeledFaceDescriptors, TNetInput, FaceMatcher, } from "@vladmandic/face-api"; import * as mime from "mime-types"; import { LocationMonitor } from "./locationMonitor"; import { getFaceDetectorOptions } from "./common"; require("@tensorflow/tfjs-node"); const { Canvas, Image, ImageData } = canvas; //@ts-ignore faceapi.env.monkeyPatch({ Canvas, Image, ImageData }); import { PLATFORM_NAME, PLUGIN_NAME } from "./settings"; import { MonitorAccessory } from "./monitorAccessory"; /** * HomebridgePlatform * This class is the main constructor for your plugin, this is where you should * parse the user config and discover/register accessories with Homebridge. */ export class HomeLocationPlatform implements DynamicPlatformPlugin { public readonly Service: typeof Service = this.api.hap.Service; public readonly Characteristic: typeof Characteristic = this.api.hap .Characteristic; // this is used to track restored cached accessories public readonly accessories: PlatformAccessory[] = []; public config: IConfig; constructor( public readonly log: Logger, config: PlatformConfig, public readonly api: API ) { this.log.debug("Finished initializing platform:", config.name); if (!isConfig(config)) { this.log.error("Configuration is incorrect or incomplete"); process.exit(1); } else { this.config = config; } this.api.on("didFinishLaunching", async () => { log.debug("Executed didFinishLaunching callback"); // run the method to discover / register your devices as accessories await this.discoverDevices(); }); } /** * This function is invoked when homebridge restores cached accessories from disk at startup. * It should be used to setup event handlers for characteristics and update respective values. */ public configureAccessory(accessory: PlatformAccessory) { this.log.info("Loading accessory from cache:", accessory.displayName); // add the restored accessory to the accessories cache so we can track if it has already been registered this.accessories.push(accessory); } /** * This is an example method showing how to register discovered accessories. * Accessories must only be registered once, previously created accessories * must not be registered again to prevent "duplicate UUID" errors. */ public async discoverDevices() { //Train facial recognition model let faceMatcher: FaceMatcher; if (this.config.trainOnStartup) { faceMatcher = await this.trainModels(); } else { const faceDetectionNet = nets.ssdMobilenetv1; await faceDetectionNet.loadFromDisk(this.config.weightDirectory); await nets.faceLandmark68Net.loadFromDisk(this.config.weightDirectory); await nets.faceRecognitionNet.loadFromDisk(this.config.weightDirectory); const raw = fs.readFileSync( path.join(this.config.trainedModelDirectory, "data.json"), "utf-8" ); faceMatcher = FaceMatcher.fromJSON(JSON.parse(raw)); } const locationMonitor = new LocationMonitor(this.config.rooms, faceMatcher); const labels = faceMatcher.labeledDescriptors.map((e) => e.label); for (const room of this.config.rooms) { for (const label of labels) { const uuid = this.api.hap.uuid.generate(room.name + label); const existingAccessory = this.accessories.find((e) => e.UUID === uuid); if (existingAccessory) { this.log.info( "Restoring existing accessory from cache: ", existingAccessory.displayName ); new MonitorAccessory(this, existingAccessory, locationMonitor); this.api.updatePlatformAccessories([existingAccessory]); } else { this.log.info("Adding new accessory:", `${room.name}+${label}`); // create a new accessory const accessory = new this.api.platformAccessory( `${room.name} ${label}`, uuid ); accessory.context["DeviceName"] = `${room.name} ${label}`; // create the accessory handler for the newly create accessory // this is imported from `platformAccessory.ts` new MonitorAccessory(this, accessory, locationMonitor); // link the accessory to your platform this.api.registerPlatformAccessories(PLUGIN_NAME, PLATFORM_NAME, [ accessory, ]); } } } } private async trainModels(): Promise { const faceDetectionNet = faceapi.nets.ssdMobilenetv1; await faceDetectionNet.loadFromDisk(this.config.weightDirectory); await faceapi.nets.faceLandmark68Net.loadFromDisk( this.config.weightDirectory ); await faceapi.nets.faceRecognitionNet.loadFromDisk( this.config.weightDirectory ); const options = getFaceDetectorOptions(faceDetectionNet); const dirs = fs.readdirSync(this.config.refImageDirectory); const refs: Array = []; for (const dir of dirs) { if ( !lstatSync(path.join(this.config.refImageDirectory, dir)).isDirectory() ) { continue; } const files = fs.readdirSync( path.join(this.config.refImageDirectory, dir) ); let referenceResults = await Promise.all( files.map(async (file: string) => { const mimeType = mime.contentType( path.extname(path.join(this.config.refImageDirectory, dir, file)) ); if (!mimeType || !mimeType.startsWith("image")) { return; } console.log(path.join(this.config.refImageDirectory, dir, file)); try { const referenceImage = (await canvas.loadImage( path.join(this.config.refImageDirectory, dir, file) )) as unknown; const descriptor = await faceapi .detectSingleFace(referenceImage as TNetInput, options) .withFaceLandmarks() .withFaceDescriptor(); if (!descriptor || !descriptor.descriptor) { throw new Error("No face found"); } const faceDescriptors = [descriptor.descriptor]; return new faceapi.LabeledFaceDescriptors(dir, faceDescriptors); } catch (err) { console.log( "An error occurred loading image at path: " + path.join(this.config.refImageDirectory, dir, file) ); } return undefined; }) ); if (referenceResults) { refs.push( ...(referenceResults.filter((e) => e) as LabeledFaceDescriptors[]) ); } } const faceMatcher = new faceapi.FaceMatcher(refs); fs.writeFile( path.join(this.config.trainedModelDirectory, "data.json"), JSON.stringify(faceMatcher.toJSON()), "utf8", (err) => { if (err) { console.log(`An error occurred while writing data model to file`); } console.log(`Successfully wrote data model to file`); } ); return faceMatcher; } }