diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..498cdb0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.github +.vscode +node_modules +compose.yaml +config/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a4d45cf --- /dev/null +++ b/.gitignore @@ -0,0 +1,179 @@ +# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore + +# Logs + +logs +_.log +npm-debug.log_ +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Caches + +.cache + +# Diagnostic reports (https://nodejs.org/api/report.html) + +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# Runtime data + +pids +_.pid +_.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover + +lib-cov + +# Coverage directory used by tools like istanbul + +coverage +*.lcov + +# nyc test coverage + +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) + +.grunt + +# Bower dependency directory (https://bower.io/) + +bower_components + +# node-waf configuration + +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) + +build/Release + +# Dependency directories + +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) + +web_modules/ + +# TypeScript cache + +*.tsbuildinfo + +# Optional npm cache directory + +.npm + +# Optional eslint cache + +.eslintcache + +# Optional stylelint cache + +.stylelintcache + +# Microbundle cache + +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history + +.node_repl_history + +# Output of 'npm pack' + +*.tgz + +# Yarn Integrity file + +.yarn-integrity + +# dotenv environment variable files + +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) + +.parcel-cache + +# Next.js build output + +.next +out + +# Nuxt.js build / generate output + +.nuxt +dist + +# Gatsby files + +# Comment in the public line in if your project uses Gatsby and not Next.js + +# https://nextjs.org/blog/next-9-1#public-directory-support + +# public + +# vuepress build output + +.vuepress/dist + +# vuepress v2.x temp and cache directory + +.temp + +# Docusaurus cache and generated files + +.docusaurus + +# Serverless directories + +.serverless/ + +# FuseBox cache + +.fusebox/ + +# DynamoDB Local files + +.dynamodb/ + +# TernJS port file + +.tern-port + +# Stores VSCode versions used for testing VSCode extensions + +.vscode-test + +# yarn v2 + +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store + +# Custom +config/*.yaml +!config/_template.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..12a3db1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +# use the official Bun image +# see all versions at https://hub.docker.com/r/oven/bun/tags +FROM oven/bun:1.1.4-alpine as base + +# install dependencies into temp directory +# this will cache them and speed up future builds +FROM base AS install +RUN mkdir -p /temp/prod +COPY package.json bun.lockb /temp/prod/ +# install with --production (exclude devDependencies) +RUN cd /temp/prod && bun install --frozen-lockfile --production + + +# final image +FROM base as release + +RUN apk add -q --progress --update --no-cache dumb-init + +WORKDIR /usr/src/app +COPY --from=install /temp/prod/node_modules node_modules +COPY . . + +# run the app +USER bun +ENV NODE_ENV=production +ENTRYPOINT [ "dumb-init" ] +CMD [ "bun", "src/index.ts" ] diff --git a/README.md b/README.md index 6c38730..20888d4 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,16 @@ -# youtube-tracker -Monitor YouTube channels +# YouTube Tracker + +Monitor YouTube channels using **RSS feeds**, +and add new videos to a **collection in Raindrop.io**. + +The applications is running in a loop, checking for new videos every X minutes (frequency). +Each run will add new videos uploaded since the last run (current time - frequency). + +## Usage + +```shell +cp config/_template.yaml config/production.yaml +# Edit config/production.yaml +docker compose build +docker compose up -d +``` diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 0000000..92b10d7 --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,21 @@ +version: '3' + +env: + DOCKER_IMAGE_NAME: thevops/youtube-tracker:v1.0.0 + +tasks: + # + # Local development + # + start: + desc: Start the app + cmds: + - bun run src/index.ts config/production.yaml + + # + # Build + # + docker-build: + desc: Build the app as Docker image + cmds: + - docker buildx build --platform linux/arm64 --tag ${DOCKER_IMAGE_NAME} --push . diff --git a/bun.lockb b/bun.lockb new file mode 100755 index 0000000..3357e10 Binary files /dev/null and b/bun.lockb differ diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..ceead62 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,14 @@ +--- +services: + app: + # image: thevops/youtube-tracker + build: + context: . + dockerfile: Dockerfile + command: + - bun + - src/index.ts + - /config/production.yaml + restart: unless-stopped + volumes: + - ./config/production.yaml:/config/production.yaml:ro diff --git a/config/_template.yaml b/config/_template.yaml new file mode 100644 index 0000000..da81db6 --- /dev/null +++ b/config/_template.yaml @@ -0,0 +1,29 @@ +log_level: info +frequency: 30 # minutes +raindrop_token: "..." +raindrop_collection_id: "..." +feeds: + - name: DevOpsToolkit + channel_id: UCfz8x0lVzJpb_dgWm9kPVrw + - name: Mateusz Chrobok + channel_id: UCTTZqMWBvLsUYqYwKTdjvkw + - name: Wolność w Remoncie + channel_id: UC_1U2qs9zaBrO4_zvvFf2BQ + - name: Klub Jegielloński + channel_id: UCtduOnyNZmXf4xZ5o56lg6w + - name: Szymon Mówi + channel_id: UCnUrMqV57fp3uPddvmDpTaA + - name: Wojna Idei + channel_id: UC7RswyY8VfbSdikz_8wdp3w + - name: Rafał Otoka + channel_id: UCo2rfn3xQF1HsrCju0-DKtw + - name: Poszukiwacz + channel_id: UCzvaTqqDP1lE0iUuoExpijw + - name: SysOpsDevOpsPolska + channel_id: UCQ8D7kwvZJxQdy70vYZCf7w + - name: UwTeamMrugalski + channel_id: UCRHXKLPXE-hYh0biKr2DGIg + - name: Sławomir Mentzen + channel_id: UCkH8DpG5uKx0YsGCFWmOQcA + - name: Program Pierwszy Polskiego YouTube'a + channel_id: UCM5zgRhkZatwLsK-Z-TyisQ diff --git a/package.json b/package.json new file mode 100644 index 0000000..1b66a01 --- /dev/null +++ b/package.json @@ -0,0 +1,18 @@ +{ + "name": "youtube-tracker", + "module": "src/index.ts", + "type": "module", + "devDependencies": { + "@types/bun": "latest", + "@types/js-yaml": "^4.0.9" + }, + "peerDependencies": { + "typescript": "^5.0.0" + }, + "dependencies": { + "js-yaml": "^4.1.0", + "raindrop-api": "https://github.com/thevops/raindrop-api#master", + "rss-parser": "^3.13.0", + "simple-fmt-logger": "https://github.com/thevops/simple-fmt-logger#v1.1.3" + } +} diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..3ff978d --- /dev/null +++ b/src/config.ts @@ -0,0 +1,48 @@ +import * as fs from "fs"; +import * as yaml from "js-yaml"; +import { SimpleFMTLogger } from "simple-fmt-logger"; +import { Raindrop } from "raindrop-api"; + +interface Feed { + name: string; + channel_id: string; +} + +interface ConfigSchema { + log_level: string; + feeds: Feed[]; + frequency: number; + raindrop_token: string; + raindrop_collection_id: string; + + [key: string]: any; +} + +function validateConfig(config: ConfigSchema) { + const requiredFields = [ + "log_level", + "feeds", + "frequency", + "raindrop_token", + "raindrop_collection_id", + ]; + + for (const field of requiredFields) { + if (!config[field]) { + throw new Error(`Missing required field: ${field}`); + } + } +} + +// Load the config file +const configPath = process.argv[2] || "config.yaml"; +export const Config = yaml.load( + fs.readFileSync(configPath, "utf8"), +) as ConfigSchema; +validateConfig(Config); + +// Initialize the logger +export const logger = new SimpleFMTLogger(Config.log_level || "info"); + +// Initialize the Raindrop API +export const raindropAPI = new Raindrop(Config.raindrop_token); diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..9ec847e --- /dev/null +++ b/src/index.ts @@ -0,0 +1,52 @@ +import { logger, Config, raindropAPI } from "./config"; +import { getNewerYouTubeVideos } from "./youtube"; + +async function main() { + logger.info("Running..."); + + for (const feed of Config.feeds) { + logger.debug(`Checking feed: ${feed.name}`); + + // Get current time minus the frequency (minutes) plus a buffer of 5 minutes + const previous_check = new Date( + Date.now() - Config.frequency * 60000 + 5 * 60000, + ); + logger.debug(`Previous check: ${previous_check.toISOString()}`); + + const new_videos = await getNewerYouTubeVideos( + previous_check, + feed.channel_id, + ); + if (new_videos.length <= 0) { + logger.debug("No new videos found"); + continue; + } + + logger.debug(`Found ${new_videos.length} new videos`); + for (const video of new_videos) { + logger.info(`Adding video to Raindrop: ${video.link}`); + const res = await raindropAPI.addItem_link( + Config.raindrop_collection_id, + video.link, + ); + if (res) { + logger.info("Video added successfully"); + } else { + logger.error("Failed to add video"); + } + } + } +} + +async function runForever() { + logger.info("Starting..."); + while (true) { + await main(); + // Sleep for X seconds + await new Promise((resolve) => + setTimeout(resolve, Config.frequency * 60 * 1000), + ); + } +} + +runForever(); diff --git a/src/youtube.ts b/src/youtube.ts new file mode 100644 index 0000000..e1ed27f --- /dev/null +++ b/src/youtube.ts @@ -0,0 +1,60 @@ +import Parser from "rss-parser"; + +interface YouTubeVideo { + title: string; + link: string; +} + +export async function getNewerYouTubeVideos( + previous_check: Date, + channel_id: string, +) { + const parser = new Parser(); + const feedUrl = `https://www.youtube.com/feeds/videos.xml?channel_id=${channel_id}`; + const feed = await parser.parseURL(feedUrl); + + // Filter out videos that were published before the previous check + // and return only the video title and link. + const new_videos: YouTubeVideo[] = feed.items + .filter((item) => { + if (item.isoDate) { + return new Date(item.isoDate) > previous_check; + } + return false; + }) + .map((item) => { + return { + title: item.title || "", + link: item.link || "", + }; + }); + + return new_videos; +} + +// ---------------------------------------------------------------------------- +// For testing purposes (bun only) +if (import.meta.main) { + function test_getNewerYouTubeVideos() { + const channel_id = "UCTTZqMWBvLsUYqYwKTdjvkw"; + const previous_check = new Date("2024-04-20T00:00:00Z"); + getNewerYouTubeVideos(previous_check, channel_id).then((new_videos) => { + console.log(new_videos); + }); + } + + function test_rssParser() { + const parser = new Parser(); + const feedUrl = + "https://www.youtube.com/feeds/videos.xml?channel_id=UCfz8x0lVzJpb_dgWm9kPVrw"; + parser.parseURL(feedUrl).then((feed) => { + console.log(feed.title); + feed.items.forEach((item) => { + console.log(item); + }); + }); + } + + // test_getNewerYouTubeVideos(); + // test_rssParser(); +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..238655f --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + // Enable latest features + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +}