From ffdf63563bab53a65b2a1a318f0f05e7917de002 Mon Sep 17 00:00:00 2001 From: Justice Almanzar Date: Tue, 15 Aug 2023 23:32:11 +0000 Subject: [PATCH] feat(plugins): Web/Vesktop AI Noise Suppression powered by RNNoise (#1477) Co-authored-by: V --- package.json | 1 + pnpm-lock.yaml | 7 + scripts/build/common.mjs | 16 +- scripts/generatePluginList.ts | 11 +- scripts/utils.mjs | 30 +++ src/plugins/rnnoise.web/icons.tsx | 21 ++ src/plugins/rnnoise.web/index.tsx | 249 +++++++++++++++++++++++ src/plugins/rnnoise.web/styles.css | 29 +++ src/utils/dependencies.ts | 4 + src/webpack/common/types/components.d.ts | 10 +- 10 files changed, 360 insertions(+), 18 deletions(-) create mode 100644 scripts/utils.mjs create mode 100644 src/plugins/rnnoise.web/icons.tsx create mode 100644 src/plugins/rnnoise.web/index.tsx create mode 100644 src/plugins/rnnoise.web/styles.css diff --git a/package.json b/package.json index cabe19a80..b8bab0ae9 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "watch": "node scripts/build/build.mjs --watch" }, "dependencies": { + "@sapphi-red/web-noise-suppressor": "0.3.3", "@vap/core": "0.0.12", "@vap/shiki": "0.10.5", "eslint-plugin-simple-header": "^1.0.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1a3c65968..5edbb11a8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,6 +9,9 @@ patchedDependencies: path: patches/eslint@8.46.0.patch dependencies: + '@sapphi-red/web-noise-suppressor': + specifier: 0.3.3 + version: 0.3.3 '@vap/core': specifier: 0.0.12 version: 0.0.12 @@ -513,6 +516,10 @@ packages: - supports-color dev: true + /@sapphi-red/web-noise-suppressor@0.3.3: + resolution: {integrity: sha512-gAC33DCXYwNTI/k1PxOVHmbbzakUSMbb/DHpoV6rn4pKZtPI1dduULSmAAm/y1ipgIlArnk2JcnQzw4n2tCZHw==} + dev: false + /@types/diff@5.0.3: resolution: {integrity: sha512-amrLbRqTU9bXMCc6uX0sWpxsQzRIo9z6MJPkH1pkez/qOxuqSZVuryJAWoBRq94CeG8JxY+VK4Le9HtjQR5T9A==} dev: true diff --git a/scripts/build/common.mjs b/scripts/build/common.mjs index 2875a9c67..b63ea6120 100644 --- a/scripts/build/common.mjs +++ b/scripts/build/common.mjs @@ -27,6 +27,7 @@ import { promisify } from "util"; // wtf is this assert syntax import PackageJSON from "../../package.json" assert { type: "json" }; +import { getPluginTarget } from "../utils.mjs"; export const VERSION = PackageJSON.version; export const BUILD_TIMESTAMP = Date.now(); @@ -81,14 +82,13 @@ export const globPlugins = kind => ({ if (file.startsWith("_") || file.startsWith(".")) continue; if (file === "index.ts") continue; - const fileBits = file.split("."); - if (fileBits.length > 2 && ["ts", "tsx"].includes(fileBits.at(-1))) { - const mod = fileBits.at(-2); - if (mod === "dev" && !watch) continue; - if (mod === "web" && kind === "discordDesktop") continue; - if (mod === "desktop" && kind === "web") continue; - if (mod === "discordDesktop" && kind !== "discordDesktop") continue; - if (mod === "vencordDesktop" && kind !== "vencordDesktop") continue; + const target = getPluginTarget(file); + if (target) { + if (target === "dev" && !watch) continue; + if (target === "web" && kind === "discordDesktop") continue; + if (target === "desktop" && kind === "web") continue; + if (target === "discordDesktop" && kind !== "discordDesktop") continue; + if (target === "vencordDesktop" && kind !== "vencordDesktop") continue; } const mod = `p${i}`; diff --git a/scripts/generatePluginList.ts b/scripts/generatePluginList.ts index c78c340da..ea08d3040 100644 --- a/scripts/generatePluginList.ts +++ b/scripts/generatePluginList.ts @@ -21,6 +21,8 @@ import { access, readFile } from "fs/promises"; import { join } from "path"; import { BigIntLiteral, createSourceFile, Identifier, isArrayLiteralExpression, isCallExpression, isExportAssignment, isIdentifier, isObjectLiteralExpression, isPropertyAccessExpression, isPropertyAssignment, isSatisfiesExpression, isStringLiteral, isVariableStatement, NamedDeclaration, NodeArray, ObjectLiteralExpression, ScriptTarget, StringLiteral, SyntaxKind } from "typescript"; +import { getPluginTarget } from "./utils.mjs"; + interface Dev { name: string; id: string; @@ -157,11 +159,10 @@ async function parseFile(fileName: string) { if (!data.name || !data.description || !data.authors) throw fail("name, description or authors are missing"); - const fileBits = fileName.split("."); - if (fileBits.length > 2 && ["ts", "tsx"].includes(fileBits.at(-1)!)) { - const mod = fileBits.at(-2)!; - if (!["web", "discordDesktop", "vencordDesktop", "dev"].includes(mod)) throw fail(`invalid target ${fileBits.at(-2)}`); - data.target = mod as any; + const target = getPluginTarget(fileName); + if (target) { + if (!["web", "discordDesktop", "vencordDesktop", "dev"].includes(target)) throw fail(`invalid target ${target}`); + data.target = target as any; } return data; diff --git a/scripts/utils.mjs b/scripts/utils.mjs new file mode 100644 index 000000000..46a946644 --- /dev/null +++ b/scripts/utils.mjs @@ -0,0 +1,30 @@ +/* + * Vencord, a modification for Discord's desktop app + * Copyright (c) 2023 Vendicated and contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/** + * @param {string} filePath + * @returns {string | null} + */ +export function getPluginTarget(filePath) { + const pathParts = filePath.split(/[/\\]/); + if (/^index\.tsx?$/.test(filePath.at(-1))) pathParts.pop(); + + const identifier = pathParts.at(-1).replace(/\.tsx?$/, ""); + const identiferBits = identifier.split("."); + return identiferBits.length === 1 ? null : identiferBits.at(-1); +} diff --git a/src/plugins/rnnoise.web/icons.tsx b/src/plugins/rnnoise.web/icons.tsx new file mode 100644 index 000000000..8fda9839a --- /dev/null +++ b/src/plugins/rnnoise.web/icons.tsx @@ -0,0 +1,21 @@ +/* + * Vencord, a modification for Discord's desktop app + * Copyright (c) 2023 Vendicated and contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +export const SupressionIcon = ({ enabled }: { enabled: boolean; }) => enabled + ? + : ; diff --git a/src/plugins/rnnoise.web/index.tsx b/src/plugins/rnnoise.web/index.tsx new file mode 100644 index 000000000..7117ca295 --- /dev/null +++ b/src/plugins/rnnoise.web/index.tsx @@ -0,0 +1,249 @@ +/* + * Vencord, a modification for Discord's desktop app + * Copyright (c) 2023 Vendicated and contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +import "./styles.css"; + +import { definePluginSettings } from "@api/Settings"; +import { classNameFactory } from "@api/Styles"; +import { Switch } from "@components/Switch"; +import { loadRnnoise, RnnoiseWorkletNode } from "@sapphi-red/web-noise-suppressor"; +import { Devs } from "@utils/constants"; +import { rnnoiseWasmSrc, rnnoiseWorkletSrc } from "@utils/dependencies"; +import { makeLazy } from "@utils/lazy"; +import { Logger } from "@utils/Logger"; +import { LazyComponent } from "@utils/react"; +import definePlugin from "@utils/types"; +import { findByCode } from "@webpack"; +import { FluxDispatcher, Popout, React } from "@webpack/common"; +import { MouseEvent, ReactNode } from "react"; + +import { SupressionIcon } from "./icons"; + +const RNNOISE_OPTION = "RNNOISE"; + +interface PanelButtonProps { + tooltipText: string; + icon: () => ReactNode; + onClick: (event: MouseEvent) => void; + tooltipClassName?: string; + disabled?: boolean; + shouldShow?: boolean; +} +const PanelButton = LazyComponent(() => findByCode("Masks.PANEL_BUTTON")); +const enum SpinnerType { + SpinningCircle = "spinningCircle", + ChasingDots = "chasingDots", + LowMotion = "lowMotion", + PulsingEllipsis = "pulsingEllipsis", + WanderingCubes = "wanderingCubes", +} +export interface SpinnerProps { + type: SpinnerType; + animated?: boolean; + className?: string; + itemClassName?: string; +} +const Spinner = LazyComponent(() => findByCode(".spinningCircleInner")); + +function createExternalStore(init: () => S) { + const subscribers = new Set<() => void>(); + let state = init(); + + return { + get: () => state, + set: (newStateGetter: (oldState: S) => S) => { + state = newStateGetter(state); + for (const cb of subscribers) cb(); + }, + use: () => { + return React.useSyncExternalStore(onStoreChange => { + subscribers.add(onStoreChange); + return () => subscribers.delete(onStoreChange); + }, () => state); + }, + } as const; +} + +const cl = classNameFactory("vc-rnnoise-"); + +const loadedStore = createExternalStore(() => ({ + isLoaded: false, + isLoading: false, + isError: false, +})); +const getRnnoiseWasm = makeLazy(() => { + loadedStore.set(s => ({ ...s, isLoading: true })); + return loadRnnoise({ + url: rnnoiseWasmSrc(), + simdUrl: rnnoiseWasmSrc(true), + }).then(buffer => { + // Check WASM magic number cus fetch doesnt throw on 4XX or 5XX + if (new DataView(buffer.slice(0, 4)).getUint32(0) !== 0x0061736D) throw buffer; + + loadedStore.set(s => ({ ...s, isLoaded: true })); + return buffer; + }).catch(error => { + if (error instanceof ArrayBuffer) error = new TextDecoder().decode(error); + logger.error("Failed to load RNNoise WASM:", error); + loadedStore.set(s => ({ ...s, isError: true })); + return null; + }).finally(() => { + loadedStore.set(s => ({ ...s, isLoading: false })); + }); +}); + +const logger = new Logger("RNNoise"); +const settings = definePluginSettings({}).withPrivateSettings<{ isEnabled: boolean; }>(); +const setEnabled = (enabled: boolean) => { + settings.store.isEnabled = enabled; + FluxDispatcher.dispatch({ type: "AUDIO_SET_NOISE_SUPPRESSION", enabled }); +}; + +function NoiseSupressionPopout() { + const { isEnabled } = settings.use(); + const { isLoading, isError } = loadedStore.use(); + const isWorking = isEnabled && !isError; + + return
+
+ Noise Supression +
+ {isLoading && } + +
+
+ Enable AI noise suppression! Make some noise—like becoming an air conditioner, or a vending machine fan—while speaking. Your friends will hear nothing but your beautiful voice ✨ +
+
; +} + +export default definePlugin({ + name: "AI Noise Suppression", + description: "Uses an open-source AI model (RNNoise) to remove background noise from your microphone", + authors: [Devs.Vap], + settings, + enabledByDefault: true, + + patches: [ + { + // Pass microphone stream to RNNoise + find: "window.webkitAudioContext", + replacement: { + match: /(?<=\i\.acquire=function\((\i)\)\{return )navigator\.mediaDevices\.getUserMedia\(\1\)(?=\})/, + replace: m => `${m}.then(stream => $self.connectRnnoise(stream))` + }, + }, + { + // Noise suppression button in call modal + find: "renderNoiseCancellation()", + replacement: { + match: /(?<=(\i)\.jsxs?.{0,70}children:\[)(?=\i\?\i\.renderNoiseCancellation\(\))/, + replace: (_, react) => `${react}.jsx($self.NoiseSupressionButton, {}),` + }, + }, + { + // Give noise suppression component a "shouldShow" prop + find: "Masks.PANEL_BUTTON", + replacement: { + match: /(?<==(\i)\.tooltipForceOpen.{0,100})(?=tooltipClassName:)/, + replace: (_, props) => `shouldShow: ${props}.shouldShow,` + } + }, + { + // Noise suppression option in voice settings + find: "Messages.USER_SETTINGS_NOISE_CANCELLATION_KRISP", + replacement: [{ + match: /(?<=(\i)=\i\?\i\.KRISP:\i.{1,20}?;)/, + replace: (_, option) => `if ($self.isEnabled()) ${option} = ${JSON.stringify(RNNOISE_OPTION)};`, + }, { + match: /(?=\i&&(\i)\.push\(\{name:(?:\i\.){1,2}Messages.USER_SETTINGS_NOISE_CANCELLATION_KRISP)/, + replace: (_, options) => `${options}.push({ name: "AI (RNNoise)", value: "${RNNOISE_OPTION}" });`, + }, { + match: /(?<=onChange:function\((\i)\)\{)(?=(?:\i\.){1,2}setNoiseCancellation)/, + replace: (_, option) => `$self.setEnabled(${option}.value === ${JSON.stringify(RNNOISE_OPTION)});`, + }], + }, + ], + + setEnabled, + isEnabled: () => settings.store.isEnabled, + async connectRnnoise(stream: MediaStream): Promise { + if (!settings.store.isEnabled) return stream; + + const audioCtx = new AudioContext(); + await audioCtx.audioWorklet.addModule(rnnoiseWorkletSrc); + + const rnnoiseWasm = await getRnnoiseWasm(); + if (!rnnoiseWasm) { + logger.warn("Failed to load RNNoise, noise suppression won't work"); + return stream; + } + + const rnnoise = new RnnoiseWorkletNode(audioCtx, { + wasmBinary: rnnoiseWasm, + maxChannels: 1, + }); + + const source = audioCtx.createMediaStreamSource(stream); + source.connect(rnnoise); + + const dest = audioCtx.createMediaStreamDestination(); + rnnoise.connect(dest); + + // Cleanup + const onEnded = () => { + rnnoise.disconnect(); + source.disconnect(); + audioCtx.close(); + rnnoise.destroy(); + }; + stream.addEventListener("inactive", onEnded, { once: true }); + + return dest.stream; + }, + NoiseSupressionButton(): ReactNode { + const { isEnabled } = settings.use(); + const { isLoading, isError } = loadedStore.use(); + + return } + spacing={8} + > + {(props, { isShown }) => ( +
+ +
} + /> + )} + ; + }, +}); diff --git a/src/plugins/rnnoise.web/styles.css b/src/plugins/rnnoise.web/styles.css new file mode 100644 index 000000000..7945c496a --- /dev/null +++ b/src/plugins/rnnoise.web/styles.css @@ -0,0 +1,29 @@ +.vc-rnnoise-popout { + background: var(--background-floating); + border-radius: 0.25em; + padding: 1em; + width: 16em; +} + +.vc-rnnoise-popout-heading { + color: var(--text-normal); + font-weight: 500; + display: flex; + justify-content: space-between; + align-items: center; + font-size: 1.1em; + margin-bottom: 1em; + gap: 0.5em; +} + +.vc-rnnoise-popout-desc { + color: var(--text-muted); + font-size: 0.9em; + display: flex; + align-items: center; + line-height: 1.5; +} + +.vc-rnnoise-tooltip { + text-align: center; +} diff --git a/src/utils/dependencies.ts b/src/utils/dependencies.ts index 3ef84b73f..f1a5c2627 100644 --- a/src/utils/dependencies.ts +++ b/src/utils/dependencies.ts @@ -79,5 +79,9 @@ const shikiWorkerDist = "https://unpkg.com/@vap/shiki-worker@0.0.8/dist"; export const shikiWorkerSrc = `${shikiWorkerDist}/${IS_DEV ? "index.js" : "index.min.js"}`; export const shikiOnigasmSrc = "https://unpkg.com/@vap/shiki@0.10.3/dist/onig.wasm"; +export const rnnoiseDist = "https://unpkg.com/@sapphi-red/web-noise-suppressor@0.3.3/dist"; +export const rnnoiseWasmSrc = (simd = false) => `${rnnoiseDist}/rnnoise${simd ? "_simd" : ""}.wasm`; +export const rnnoiseWorkletSrc = `${rnnoiseDist}/rnnoise/workletProcessor.js`; + // @ts-expect-error SHUT UP export const getStegCloak = makeLazy(() => import("https://unpkg.com/stegcloak-dist@1.0.0/index.js")); diff --git a/src/webpack/common/types/components.d.ts b/src/webpack/common/types/components.d.ts index d6d19fedc..4b316aad6 100644 --- a/src/webpack/common/types/components.d.ts +++ b/src/webpack/common/types/components.d.ts @@ -17,7 +17,7 @@ */ import type { Moment } from "moment"; -import type { ComponentType, CSSProperties, FunctionComponent, HtmlHTMLAttributes, HTMLProps, PropsWithChildren, PropsWithRef, ReactNode, Ref } from "react"; +import type { ComponentType, CSSProperties, FunctionComponent, HtmlHTMLAttributes, HTMLProps, KeyboardEvent, MouseEvent, PropsWithChildren, PropsWithRef, ReactNode, Ref } from "react"; export type TextVariant = "heading-sm/normal" | "heading-sm/medium" | "heading-sm/semibold" | "heading-sm/bold" | "heading-md/normal" | "heading-md/medium" | "heading-md/semibold" | "heading-md/bold" | "heading-lg/normal" | "heading-lg/medium" | "heading-lg/semibold" | "heading-lg/bold" | "heading-xl/normal" | "heading-xl/medium" | "heading-xl/bold" | "heading-xxl/normal" | "heading-xxl/medium" | "heading-xxl/bold" | "eyebrow" | "heading-deprecated-14/normal" | "heading-deprecated-14/medium" | "heading-deprecated-14/bold" | "text-xxs/normal" | "text-xxs/medium" | "text-xxs/semibold" | "text-xxs/bold" | "text-xs/normal" | "text-xs/medium" | "text-xs/semibold" | "text-xs/bold" | "text-sm/normal" | "text-sm/medium" | "text-sm/semibold" | "text-sm/bold" | "text-md/normal" | "text-md/medium" | "text-md/semibold" | "text-md/bold" | "text-lg/normal" | "text-lg/medium" | "text-lg/semibold" | "text-lg/bold" | "display-sm" | "display-md" | "display-lg" | "code"; export type FormTextTypes = Record<"DEFAULT" | "INPUT_PLACEHOLDER" | "DESCRIPTION" | "LABEL_BOLD" | "LABEL_SELECTED" | "LABEL_DESCRIPTOR" | "ERROR" | "SUCCESS", string>; @@ -338,16 +338,16 @@ export type Popout = ComponentType<{ thing: { "aria-controls": string; "aria-expanded": boolean; - onClick(event: MouseEvent): void; - onKeyDown(event: KeyboardEvent): void; - onMouseDown(event: MouseEvent): void; + onClick(event: MouseEvent): void; + onKeyDown(event: KeyboardEvent): void; + onMouseDown(event: MouseEvent): void; }, data: { isShown: boolean; position: string; } ): ReactNode; - shouldShow: boolean; + shouldShow?: boolean; renderPopout(args: { closePopout(): void; isPositioned: boolean;