diff --git a/docs/new-tool.md b/docs/new-tool.md index d8ab13aab9..5ebc3ac9b1 100644 --- a/docs/new-tool.md +++ b/docs/new-tool.md @@ -164,6 +164,8 @@ Containerbase then removes the shell wrappers, the current-version marker and th That default is only correct if `install()` writes **everything** below `createVersionedToolPath()`. If your tool writes anywhere else - `$HOME`, a shared cache, or system packages - either override `uninstall()` to clean that up, or raise it on the Issue so we can agree what should happen. +If the tool keeps a download cache in the containerbase cache, eg. a folder its prepare service links from `$HOME`, declare it in `cacheDirs`, so `containerbase-cli cleanup cache` removes it, see the [tool installer best practices](./tool-installer-best-practices.md#folders-ownership-and-links). + Two more things worth knowing: - If you attempt to uninstall a tool that other installed tools depend on (by declaring it as a `parent`), Containerbase will refuse to uninstall until `--recursive` is passed diff --git a/docs/tool-installer-best-practices.md b/docs/tool-installer-best-practices.md index 1edd3090c3..3b9744bde4 100644 --- a/docs/tool-installer-best-practices.md +++ b/docs/tool-installer-best-practices.md @@ -66,6 +66,16 @@ override async prepare(): Promise { } ``` +Declare the folders in the cache which only hold data the tool downloads again, like a package cache, in `cacheDirs` on the install service, relative to the cache path. +[`containerbase-cli cleanup cache`](./tools.md#cleanup-cache-command) removes them, so never list config files or folders holding them, and list a sub folder where needed: + +```ts +override readonly cacheDirs = ['.cargo/git', '.cargo/registry']; +``` + +Caches in the containerbase user's `~/.cache`, the `.cache` folder in the cache path, need no entry, its contents are always removed. +Root's `/root/.cache` isn't touched. + See [`src/cli/services/path.service.ts`](../src/cli/services/path.service.ts) for the full set of path helpers before reaching for `node:fs` directly. ## Shared helpers diff --git a/docs/tools.md b/docs/tools.md index 143084d3f5..ce31a82b6d 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -17,6 +17,7 @@ - [List](#list) - [list-tools](#list-tools-command) - [Cleanup](#cleanup) + - [cleanup-cache](#cleanup-cache-command) - [cleanup-downloads](#cleanup-downloads-command) --- @@ -525,6 +526,40 @@ When writing to a file, JSON output is minified. ## Cleanup +### `containerbase-cli cleanup cache` + +#### Description + +Removes the known tool caches from `/tmp/containerbase/cache`. + +These are the cache folders of every supported tool, whether it is installed or not, eg. the npm cache in `.npm/_cacache`, the pnpm store in `.local/share/pnpm/store`, the maven repository in `.m2/repository` or the go module cache in `go/pkg/mod`. +The contents of the generic `.cache` folder and the virtualenv cache in `.local/share/virtualenv` are removed too. +Config files, like `.npmrc`, `.m2/settings.xml` or the conan profiles, are kept. +When a tool's cache folder or `.cache` itself is a link, eg. to a volume, its target is emptied and the link kept, a link to a missing target is skipped. +Entries inside `.cache` are removed as they are, so a link there is removed, not its target. + +The download cache in `CONTAINERBASE_CACHE_DIR` is left alone, use [`cleanup downloads`](#cleanup-downloads-command) for it. +A cache folder or `.cache` entry holding it, eg. `.cache/containerbase`, is skipped. + +With `--dry-run` nothing is removed and every path which would be removed is logged. + +#### Usage + +`$ containerbase-cli cleanup cache [-d,--dry-run]` + +#### Examples + +- Removes all known tool caches + ```bash + containerbase-cli cleanup cache + ``` +- Lists the caches which would be removed + ```bash + containerbase-cli cleanup cache --dry-run + ``` + +
+ ### `containerbase-cli cleanup downloads` #### Description diff --git a/src/cli/cleanup/cleanup-cache.service.spec.ts b/src/cli/cleanup/cleanup-cache.service.spec.ts new file mode 100644 index 0000000000..a158ccaeb9 --- /dev/null +++ b/src/cli/cleanup/cleanup-cache.service.spec.ts @@ -0,0 +1,316 @@ +import fs from 'node:fs/promises'; +import { join } from 'node:path'; +import { injectFromHierarchy, injectable } from 'inversify'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { BaseInstallService } from '../install-tool/base-install.service.ts'; +import { INSTALL_TOOL_TOKEN } from '../install-tool/install-tool.service.ts'; +import { PathService, createContainer } from '../services/index.ts'; +import { logger, pathExists } from '../utils/index.ts'; +import { CleanupCacheService } from './cleanup-cache.service.ts'; +import { cachePath, rootPath } from '~test/path.ts'; + +@injectable() +@injectFromHierarchy() +abstract class TestInstallService extends BaseInstallService { + /** Does nothing, for tests. */ + override install(_version: string): Promise { + return Promise.resolve(); + } + + /** Does nothing, for tests. */ + override link(_version: string): Promise { + return Promise.resolve(); + } +} + +/** a tool with a cache folder */ +@injectable() +@injectFromHierarchy() +class CachingInstallService extends TestInstallService { + override readonly name = 'caching'; + + override readonly cacheDirs = ['.caching/cache', '.shared/cache']; +} + +/** another tool sharing a cache folder */ +@injectable() +@injectFromHierarchy() +class SharingInstallService extends TestInstallService { + override readonly name = 'sharing'; + + override readonly cacheDirs = ['.shared/cache']; +} + +/** a tool without cache folders */ +@injectable() +@injectFromHierarchy() +class PlainInstallService extends TestInstallService { + override readonly name = 'plain'; +} + +/** Creates a file and its missing parent folders. */ +async function touch(path: string): Promise { + await fs.mkdir(join(path, '..'), { recursive: true }); + await fs.writeFile(path, ''); +} + +describe('cli/cleanup/cleanup-cache.service', () => { + let tmpCache: string; + let svc: CleanupCacheService; + + beforeEach(async () => { + tmpCache = rootPath('tmp/containerbase/cache'); + await fs.rm(rootPath('tmp'), { recursive: true, force: true }); + await fs.rm(rootPath('var/cache'), { recursive: true, force: true }); + await fs.rm(rootPath('volume'), { recursive: true, force: true }); + await fs.rm(cachePath(''), { recursive: true, force: true }); + await fs.mkdir(cachePath(''), { recursive: true }); + + const container = createContainer(); + container.bind(CleanupCacheService).toSelf(); + container.bind(INSTALL_TOOL_TOKEN).to(CachingInstallService); + container.bind(INSTALL_TOOL_TOKEN).to(SharingInstallService); + container.bind(INSTALL_TOOL_TOKEN).to(PlainInstallService); + svc = await container.getAsync(CleanupCacheService); + }); + + test('uses the containerbase cache', async () => { + const pathSvc = await createContainer().getAsync(PathService); + expect(pathSvc.cachePath).toBe(tmpCache); + }); + + test('removes the known caches and keeps the rest', async () => { + await touch(join(tmpCache, '.caching/cache/file')); + await touch(join(tmpCache, '.caching/config')); + await touch(join(tmpCache, '.shared/cache/file')); + await touch(join(tmpCache, '.cache/pip/file')); + await touch(join(tmpCache, '.cache/.hidden')); + await touch(join(tmpCache, '.local/share/virtualenv/file')); + await touch(join(tmpCache, '.local/share/other')); + await touch(cachePath('download/file')); + // a dangling link counts as well + await fs.symlink('missing', join(tmpCache, '.cache/link')); + + expect(await svc.cleanupCache()).toEqual([ + join(tmpCache, '.cache/.hidden'), + join(tmpCache, '.cache/link'), + join(tmpCache, '.cache/pip'), + join(tmpCache, '.caching/cache'), + join(tmpCache, '.local/share/virtualenv'), + join(tmpCache, '.shared/cache'), + ]); + + expect(await fs.readdir(join(tmpCache, '.cache'))).toEqual([]); + // the download cache is left alone + expect(await pathExists(cachePath('download/file'))).toBe(true); + expect(await fs.readdir(join(tmpCache, '.caching'))).toEqual(['config']); + expect(await fs.readdir(join(tmpCache, '.local/share'))).toEqual(['other']); + expect(await pathExists(join(tmpCache, '.shared/cache'))).toBe(false); + expect(logger.debug).toHaveBeenCalledWith( + { path: join(tmpCache, '.caching/cache') }, + 'remove cache', + ); + }); + + test('removes the caches when the download cache is missing', async () => { + await touch(join(tmpCache, '.cache/pip/file')); + vi.stubEnv('CONTAINERBASE_CACHE_DIR', rootPath('missing-downloads')); + + expect(await svc.cleanupCache()).toEqual([join(tmpCache, '.cache/pip')]); + }); + + test('skips paths holding the download cache', async () => { + await touch(join(tmpCache, '.cache/containerbase/download/file')); + await touch(join(tmpCache, '.cache/pip/file')); + vi.stubEnv( + 'CONTAINERBASE_CACHE_DIR', + join(tmpCache, '.cache/containerbase/download'), + ); + + expect(await svc.cleanupCache()).toEqual([join(tmpCache, '.cache/pip')]); + + expect( + await pathExists(join(tmpCache, '.cache/containerbase/download/file')), + ).toBe(true); + expect(logger.debug).toHaveBeenCalledWith( + { path: join(tmpCache, '.cache/containerbase') }, + 'cache holds the download cache', + ); + + // through a link, the download cache in the link target + await touch(rootPath('volume/downloads/file')); + await fs.mkdir(join(tmpCache, '.shared'), { recursive: true }); + await fs.symlink(rootPath('volume'), join(tmpCache, '.shared/cache')); + vi.stubEnv('CONTAINERBASE_CACHE_DIR', rootPath('volume/downloads')); + expect(await svc.cleanupCache()).toEqual([ + join(tmpCache, '.cache/containerbase'), + ]); + expect(await pathExists(rootPath('volume/downloads/file'))).toBe(true); + }); + + test('dry run', async () => { + await touch(join(tmpCache, '.caching/cache/file')); + + expect(await svc.cleanupCache(true)).toEqual([ + join(tmpCache, '.caching/cache'), + ]); + + expect(await pathExists(join(tmpCache, '.caching/cache/file'))).toBe(true); + expect(logger.info).toHaveBeenCalledWith( + `Dry run: remove cache ${join(tmpCache, '.caching/cache')} ...`, + ); + }); + + test('works without any cache', async () => { + expect(await svc.cleanupCache()).toEqual([]); + }); + + test('empties a linked cache folder and keeps the link', async () => { + const volume = rootPath('volume/caching'); + await touch(join(volume, 'file')); + await touch(join(volume, 'dir/file')); + await touch(join(rootPath('volume/cache'), 'pip/file')); + await fs.mkdir(join(tmpCache, '.caching'), { recursive: true }); + await fs.symlink(volume, join(tmpCache, '.caching/cache')); + // the generic `.cache` folder is emptied through its link too + await fs.symlink(rootPath('volume/cache'), join(tmpCache, '.cache')); + + expect(await svc.cleanupCache()).toEqual([ + join(tmpCache, '.cache/pip'), + join(tmpCache, '.caching/cache'), + ]); + + expect(await fs.readlink(join(tmpCache, '.caching/cache'))).toBe(volume); + expect(await fs.readdir(volume)).toEqual([]); + expect(await fs.readdir(rootPath('volume/cache'))).toEqual([]); + }); + + test('finds the caches in a configured tmp dir without the link', async () => { + vi.stubEnv('CONTAINERBASE_TMP_DIR', rootPath('var/cache/containerbase')); + await fs.mkdir(rootPath('tmp'), { recursive: true }); + await touch(rootPath('var/cache/containerbase/cache/.caching/cache/file')); + + expect(await svc.cleanupCache()).toEqual([ + join(tmpCache, '.caching/cache'), + ]); + + expect(await fs.readlink(rootPath('tmp/containerbase'))).toBe( + rootPath('var/cache/containerbase'), + ); + expect( + await pathExists( + rootPath('var/cache/containerbase/cache/.caching/cache'), + ), + ).toBe(false); + }); + + test('dry run reads a configured tmp dir without the link and changes nothing', async () => { + vi.stubEnv('CONTAINERBASE_TMP_DIR', rootPath('var/cache/containerbase')); + await fs.mkdir(rootPath('tmp'), { recursive: true }); + const cache = rootPath('var/cache/containerbase/cache'); + await touch(join(cache, '.caching/cache/file')); + await touch(join(cache, '.cache/pip/file')); + + expect(await svc.cleanupCache(true)).toEqual([ + join(cache, '.cache/pip'), + join(cache, '.caching/cache'), + ]); + + expect(await fs.readdir(rootPath('tmp'))).toEqual([]); + expect(await pathExists(join(cache, '.caching/cache/file'))).toBe(true); + expect(await pathExists(join(cache, '.cache/pip/file'))).toBe(true); + }); + + test('dry run reads the default cache when the link exists', async () => { + vi.stubEnv('CONTAINERBASE_TMP_DIR', rootPath('var/cache/containerbase')); + await fs.mkdir(rootPath('tmp'), { recursive: true }); + await touch(rootPath('var/cache/containerbase/cache/.caching/cache/file')); + await fs.symlink( + rootPath('var/cache/containerbase'), + rootPath('tmp/containerbase'), + ); + + expect(await svc.cleanupCache(true)).toEqual([ + join(tmpCache, '.caching/cache'), + ]); + }); + + test('skips a linked cache folder with a missing target', async () => { + await fs.mkdir(join(tmpCache, '.caching'), { recursive: true }); + await fs.symlink( + rootPath('volume/missing'), + join(tmpCache, '.caching/cache'), + ); + + expect(await svc.cleanupCache()).toEqual([]); + + expect(await fs.readlink(join(tmpCache, '.caching/cache'))).toBe( + rootPath('volume/missing'), + ); + expect(logger.debug).toHaveBeenCalledWith( + { path: join(tmpCache, '.caching/cache') }, + 'cache link target missing', + ); + }); + + test('does not make a link target writable', async () => { + const target = rootPath('volume/readonly'); + await fs.mkdir(target, { recursive: true }); + await fs.chmod(target, 0o555); + await fs.mkdir(join(tmpCache, '.cache'), { recursive: true }); + await fs.symlink(target, join(tmpCache, '.cache/link')); + vi.spyOn(fs, 'rm').mockRejectedValueOnce( + Object.assign(new Error('EACCES'), { code: 'EACCES' }), + ); + + await expect(svc.cleanupCache()).rejects.toThrow( + 'Failed to remove 1 cache paths', + ); + + expect((await fs.stat(target)).mode & 0o777).toBe(0o555); + await fs.chmod(target, 0o755); + }); + + test('makes a folder writable when it cannot be removed', async () => { + await touch(join(tmpCache, '.caching/cache/file')); + vi.spyOn(fs, 'rm').mockRejectedValueOnce( + Object.assign(new Error('EACCES'), { code: 'EACCES' }), + ); + + expect(await svc.cleanupCache()).toEqual([ + join(tmpCache, '.caching/cache'), + ]); + + expect(await pathExists(join(tmpCache, '.caching/cache'))).toBe(false); + }); + + test('removes read-only folders', async () => { + const dir = join(tmpCache, '.caching/cache/mod'); + await touch(join(dir, 'file')); + await fs.chmod(dir, 0o555); + + expect(await svc.cleanupCache()).toEqual([ + join(tmpCache, '.caching/cache'), + ]); + + expect(await pathExists(join(tmpCache, '.caching/cache'))).toBe(false); + }); + + test('removes the other paths when one fails and throws at the end', async () => { + await touch(join(tmpCache, '.caching/cache/file')); + await touch(join(tmpCache, '.shared/cache/file')); + const err = Object.assign(new Error('EBUSY'), { code: 'EBUSY' }); + vi.spyOn(fs, 'rm').mockRejectedValueOnce(err); + + await expect(svc.cleanupCache()).rejects.toThrow( + 'Failed to remove 1 cache paths', + ); + + expect(logger.warn).toHaveBeenCalledExactlyOnceWith( + { path: join(tmpCache, '.caching/cache'), err }, + 'cannot remove cache', + ); + expect(await pathExists(join(tmpCache, '.caching/cache/file'))).toBe(true); + expect(await pathExists(join(tmpCache, '.shared/cache'))).toBe(false); + }); +}); diff --git a/src/cli/cleanup/cleanup-cache.service.ts b/src/cli/cleanup/cleanup-cache.service.ts new file mode 100644 index 0000000000..7f70a29513 --- /dev/null +++ b/src/cli/cleanup/cleanup-cache.service.ts @@ -0,0 +1,158 @@ +import fs from 'node:fs/promises'; +import { join, sep } from 'node:path'; +import { inject, injectable, multiInject, optional } from 'inversify'; +import type { BaseInstallService } from '../install-tool/base-install.service.ts'; +import { INSTALL_TOOL_TOKEN } from '../install-tool/install-tool.service.ts'; +import { EnvService, PathService } from '../services/index.ts'; +import { logger, spawn } from '../utils/index.ts'; + +/** The paths of the entries in a folder, none for a missing folder. */ +async function listEntries(dir: string): Promise { + const entries = await fs.readdir(dir).catch(() => []); + return entries.map((entry) => join(dir, entry)); +} + +/** Whether something exists at the path, also a dangling link. */ +async function exists(path: string): Promise { + return !!(await fs.lstat(path).catch(() => null)); +} + +/** Whether the path, with its links resolved, is the folder or a parent of it. */ +async function contains(path: string, folder: string): Promise { + const real = await fs.realpath(path).catch(() => null); + return !!real && (folder === real || folder.startsWith(`${real}${sep}`)); +} + +/** + * Removes a file or folder. Folders which can't be removed because of their + * permissions, eg. the read-only go module cache, are made writable first. + * A link is never made writable, `chmod` would change its target. + */ +async function remove(path: string): Promise { + try { + await fs.rm(path, { recursive: true, force: true }); + } catch (err) { + if ( + (err as NodeJS.ErrnoException).code !== 'EACCES' || + (await fs.lstat(path)).isSymbolicLink() + ) { + throw err; + } + await spawn('chmod', ['-R', 'u+w', path]); + await fs.rm(path, { recursive: true, force: true }); + } +} + +/** + * Removes a cache folder. For a link only the contents of its target are + * removed, so a redirect, eg. to a volume, is kept. + */ +async function removeCacheDir(path: string): Promise { + if (!(await fs.lstat(path)).isSymbolicLink()) { + await remove(path); + return; + } + for (const entry of await listEntries(path)) { + await remove(entry); + } +} + +@injectable() +export class CleanupCacheService { + @multiInject(INSTALL_TOOL_TOKEN) + @optional() + private readonly toolSvcs: BaseInstallService[] = []; + @inject(PathService) + private readonly pathSvc!: PathService; + @inject(EnvService) + private readonly envSvc!: EnvService; + + /** + * Removes the cache folders of all tools, see `BaseInstallService.cacheDirs`, + * whether they are installed or not, the virtualenv cache and the contents + * of the generic `.cache` folder, which is kept, as a link points to it. + * A cache folder which is a link is emptied instead, so its target is kept, + * a link to a missing target is skipped. Links inside `.cache` are removed. + * The download cache is left alone, see `CacheService.evict`, also a path + * holding it, eg. `.cache/containerbase`, is skipped. + * + * The cache path is linked to a configured `CONTAINERBASE_TMP_DIR` first, + * like an install does, so caches there are found when the link is missing, + * see `PathService.ensureTmpLink`. A dry run doesn't link, it reads the + * configured folder then. + * + * A path which can't be removed is logged and the others are still removed. + * With `dryRun` every path which would be removed is logged. + * + * @returns the removed paths, with `dryRun` the paths which would be removed + * @throws when any path could not be removed + */ + async cleanupCache(dryRun = false): Promise { + const cachePath = await this.findCachePath(dryRun); + const roots = new Set( + this.toolSvcs.flatMap((svc) => + svc.cacheDirs.map((dir) => join(cachePath, dir)), + ), + ); + roots.add(join(cachePath, '.local/share/virtualenv')); + const paths = new Set(roots); + for (const path of await listEntries(join(cachePath, '.cache'))) { + paths.add(path); + } + + const { cacheDir } = this.envSvc; + const downloads = + cacheDir && (await fs.realpath(cacheDir).catch(() => null)); + + const removed: string[] = []; + let failed = 0; + for (const path of Array.from(paths).sort()) { + if (!(await exists(path))) { + continue; + } + if (downloads && (await contains(path, downloads))) { + logger.debug({ path }, 'cache holds the download cache'); + continue; + } + if (roots.has(path) && !(await fs.stat(path).catch(() => null))) { + logger.debug({ path }, 'cache link target missing'); + continue; + } + if (dryRun) { + logger.info(`Dry run: remove cache ${path} ...`); + } else { + logger.debug({ path }, 'remove cache'); + try { + await (roots.has(path) ? removeCacheDir(path) : remove(path)); + } catch (err) { + logger.warn({ path, err }, 'cannot remove cache'); + failed++; + continue; + } + } + removed.push(path); + } + if (failed) { + throw new Error(`Failed to remove ${failed} cache paths`); + } + return removed; + } + + /** + * The cache path, linked to a configured `CONTAINERBASE_TMP_DIR` first, see + * `PathService.ensureTmpLink`. A dry run changes nothing, so with a + * configured folder and a missing link it reads the cache below the + * configured folder instead. + */ + private async findCachePath(dryRun: boolean): Promise { + const { cachePath, defaultTmpDir, tmpDir } = this.pathSvc; + if (!dryRun) { + await this.pathSvc.ensureTmpLink(); + return cachePath; + } + if (tmpDir !== defaultTmpDir && !(await exists(defaultTmpDir))) { + return join(tmpDir, 'cache'); + } + return cachePath; + } +} diff --git a/src/cli/cleanup/index.spec.ts b/src/cli/cleanup/index.spec.ts index e61f7b3b4b..8d98cb758d 100644 --- a/src/cli/cleanup/index.spec.ts +++ b/src/cli/cleanup/index.spec.ts @@ -1,9 +1,51 @@ -import { describe, expect, test, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import { join } from 'node:path'; +import { beforeAll, describe, expect, test, vi } from 'vitest'; import { CacheService } from '../services/index.ts'; -import { logger } from '../utils/index.ts'; -import { cleanupDownloads } from './index.ts'; +import { logger, pathExists } from '../utils/index.ts'; +import { cleanupCache, cleanupDownloads } from './index.ts'; +import { ensurePaths, rootPath } from '~test/path.ts'; describe('cli/cleanup/index', () => { + beforeAll(async () => { + await ensurePaths([ + 'tmp/containerbase/cache', + 'usr/local/containerbase/tools/v2', + 'var/lib/containerbase', + ]); + }); + + describe('cleanupCache', () => { + test('removes the caches of the registered tools', async () => { + const cache = rootPath('tmp/containerbase/cache/.cargo/registry'); + await fs.mkdir(cache, { recursive: true }); + + const dryRun = await cleanupCache(true); + expect(dryRun).toContain(cache); + expect(await pathExists(cache)).toBe(true); + expect(logger.info).toHaveBeenCalledWith( + `Dry run: would remove ${dryRun.length} cache paths`, + ); + + const removed = await cleanupCache(); + expect(removed).toContain(cache); + expect(await pathExists(cache)).toBe(false); + expect(logger.info).toHaveBeenCalledWith( + `Removed ${removed.length} cache paths`, + ); + }); + + test('removes the pnpm store and keeps the global pnpm folder', async () => { + const pnpm = rootPath('tmp/containerbase/cache/.local/share/pnpm'); + await fs.mkdir(join(pnpm, 'store/v10'), { recursive: true }); + await fs.mkdir(join(pnpm, 'global'), { recursive: true }); + + expect(await cleanupCache()).toContain(join(pnpm, 'store')); + + expect(await fs.readdir(pnpm)).toEqual(['global']); + }); + }); + describe('cleanupDownloads', () => { test('evicts the download cache', async () => { const spy = vi diff --git a/src/cli/cleanup/index.ts b/src/cli/cleanup/index.ts index d8655d2ff0..43b40daf07 100644 --- a/src/cli/cleanup/index.ts +++ b/src/cli/cleanup/index.ts @@ -1,5 +1,26 @@ +import { prepareInstallContainer } from '../install-tool/index.ts'; import { CacheService, createContainer } from '../services/index.ts'; import { logger } from '../utils/index.ts'; +import { CleanupCacheService } from './cleanup-cache.service.ts'; + +/** + * Removes the known tool caches, see `CleanupCacheService.cleanupCache`, and + * logs how many were removed. + * + * @returns the removed paths, with `dryRun` the paths which would be removed + */ +export async function cleanupCache(dryRun = false): Promise { + const container = await prepareInstallContainer(); + container.bind(CleanupCacheService).toSelf(); + const svc = await container.getAsync(CleanupCacheService); + const removed = await svc.cleanupCache(dryRun); + logger.info( + dryRun + ? `Dry run: would remove ${removed.length} cache paths` + : `Removed ${removed.length} cache paths`, + ); + return removed; +} /** * Evicts the oldest files from the download cache, see `CacheService.evict`, diff --git a/src/cli/command/cleanup-cache.spec.ts b/src/cli/command/cleanup-cache.spec.ts new file mode 100644 index 0000000000..ce4398837d --- /dev/null +++ b/src/cli/command/cleanup-cache.spec.ts @@ -0,0 +1,50 @@ +import { describe, expect, test, vi } from 'vitest'; +import { logger } from '../utils/index.ts'; +import { testCli } from '~test/di.ts'; + +const mocks = vi.hoisted(() => ({ + cleanupCache: vi.fn(), +})); + +vi.mock('../cleanup/index.ts', () => mocks); + +describe('cli/command/cleanup-cache', () => { + test('works', async () => { + const cli = testCli('containerbase-cli'); + mocks.cleanupCache.mockResolvedValueOnce(['/tmp/a', '/tmp/b']); + + expect(await cli.run(['cleanup', 'cache'])).toBe(0); + + expect(mocks.cleanupCache).toHaveBeenCalledExactlyOnceWith(false); + expect(logger.info).toHaveBeenCalledWith( + expect.stringContaining('Cleanup cache succeeded'), + ); + }); + + test('dry run', async () => { + const cli = testCli('containerbase-cli'); + mocks.cleanupCache.mockResolvedValueOnce(['/tmp/a']); + + expect(await cli.run(['cleanup', 'cache', '-d'])).toBe(0); + + expect(mocks.cleanupCache).toHaveBeenCalledExactlyOnceWith(true); + }); + + test('fails', async () => { + const cli = testCli('containerbase-cli'); + mocks.cleanupCache.mockRejectedValueOnce(new Error('test')); + + expect(await cli.run(['cleanup', 'cache'])).toBe(1); + + expect(logger.error).toHaveBeenCalledWith('test'); + expect(logger.fatal).toHaveBeenCalledWith( + expect.stringContaining('Cleanup cache failed'), + ); + + // a rejection which is not an `Error` has no message to report + mocks.cleanupCache.mockRejectedValueOnce('boom'); + expect(await cli.run(['cleanup', 'cache'])).toBe(1); + expect(logger.debug).toHaveBeenCalledWith('boom'); + expect(logger.error).not.toHaveBeenCalledWith('boom'); + }); +}); diff --git a/src/cli/command/cleanup-cache.ts b/src/cli/command/cleanup-cache.ts new file mode 100644 index 0000000000..e768105494 --- /dev/null +++ b/src/cli/command/cleanup-cache.ts @@ -0,0 +1,50 @@ +import { Command, Option } from 'clipanion'; +import prettyMilliseconds from 'pretty-ms'; +import { cleanupCache } from '../cleanup/index.ts'; +import { logger } from '../utils/index.ts'; +import { command } from './utils.ts'; + +@command('containerbase-cli') +export class CleanupCacheCommand extends Command { + static override paths = [['cleanup', 'cache']]; + + static override usage = Command.Usage({ + description: + 'Removes the known tool caches from `/tmp/containerbase/cache`.', + examples: [ + ['Removes all known tool caches', '$0 cleanup cache'], + ['Lists the caches which would be removed', '$0 cleanup cache --dry-run'], + ], + }); + + dryRun = Option.Boolean('-d,--dry-run', false); + + /** Removes the known tool caches, returning a non-zero exit code on failure. */ + async execute(): Promise { + const start = Date.now(); + let error = false; + logger.info('Cleanup cache ...'); + try { + await cleanupCache(this.dryRun); + return 0; + } catch (err) { + error = true; + logger.debug(err); + if (err instanceof Error) { + logger.error(err.message); + } + return 1; + /* v8 ignore next -- coverage bug */ + } finally { + if (error) { + logger.fatal( + `Cleanup cache failed in ${prettyMilliseconds(Date.now() - start)}.`, + ); + } else { + logger.info( + `Cleanup cache succeeded in ${prettyMilliseconds(Date.now() - start)}.`, + ); + } + } + } +} diff --git a/src/cli/command/index.ts b/src/cli/command/index.ts index 8a69ec460b..56e53c9dd5 100644 --- a/src/cli/command/index.ts +++ b/src/cli/command/index.ts @@ -1,3 +1,4 @@ +import './cleanup-cache.ts'; import './cleanup-downloads.ts'; import './cleanup-path.ts'; import './file-download.ts'; diff --git a/src/cli/install-tool/base-install.service.ts b/src/cli/install-tool/base-install.service.ts index b54254b6a9..e499248e9b 100644 --- a/src/cli/install-tool/base-install.service.ts +++ b/src/cli/install-tool/base-install.service.ts @@ -72,6 +72,13 @@ export abstract class BaseInstallService { */ readonly canUninstall: boolean = true; + /** + * The cache folders of the tool, relative to `PathService.cachePath`, eg. + * `.cargo/registry`, which `cleanup cache` removes. Only folders holding + * data the tool downloads again, never config files. + */ + readonly cacheDirs: readonly string[] = []; + /** * The package manager installing the tool, published as `type` in the base * package metadata. diff --git a/src/cli/install-tool/index.ts b/src/cli/install-tool/index.ts index 9ae6a81866..0f0ff4533e 100644 --- a/src/cli/install-tool/index.ts +++ b/src/cli/install-tool/index.ts @@ -107,7 +107,7 @@ import { * every v2 shell tool. The generic ones are bound last, so a modern service * wins over a custom v2 shell tool with the same name. */ -async function prepareInstallContainer(): Promise { +export async function prepareInstallContainer(): Promise { logger.trace('preparing install container'); const container = createContainer(); diff --git a/src/cli/tools/dart/index.ts b/src/cli/tools/dart/index.ts index 6425d21678..bcdcbaf051 100644 --- a/src/cli/tools/dart/index.ts +++ b/src/cli/tools/dart/index.ts @@ -8,6 +8,7 @@ import { initPubCache, prepareDartHome, preparePubCache, + pubCacheDirs, } from './utils.ts'; // Dart SDK sample urls @@ -44,6 +45,12 @@ export class DartInstallService extends BaseInstallService { /** The install rejects older majors, 2.0.0 is container tested. */ override readonly minVersion = '2.0.0'; + /** + * The packages pub downloaded, globally activated packages in the pub cache + * are kept. + */ + override readonly cacheDirs = pubCacheDirs; + /** The architecture name used by the dart sdk archives. */ private get arch(): string { switch (this.envSvc.arch) { diff --git a/src/cli/tools/dart/utils.ts b/src/cli/tools/dart/utils.ts index 49131905ed..1710a608ea 100644 --- a/src/cli/tools/dart/utils.ts +++ b/src/cli/tools/dart/utils.ts @@ -3,6 +3,13 @@ import { join } from 'node:path'; import type { EnvService, PathService } from '../../services'; import { pathExists } from '../../utils/index.ts'; +/** + * The package folders of the pub cache, relative to the containerbase cache. + * Globally activated packages in `.pub-cache/bin` and + * `.pub-cache/global_packages` are no cache. + */ +export const pubCacheDirs = ['.pub-cache/git', '.pub-cache/hosted']; + /** * Links the user's `.dart` and `.dart-tool` folders to the cache, keeping any * existing link, and turns off the dart analytics for root. diff --git a/src/cli/tools/dotnet/index.ts b/src/cli/tools/dotnet/index.ts index ce95804712..0ba77738b3 100644 --- a/src/cli/tools/dotnet/index.ts +++ b/src/cli/tools/dotnet/index.ts @@ -94,6 +94,16 @@ export class DotnetInstallService extends BaseInstallService { */ override readonly minVersion = '6.0.100'; + /** + * The global packages folder and the nuget http caches, the nuget config in + * `.nuget/NuGet` is kept. + */ + override readonly cacheDirs = [ + '.local/share/NuGet/http-cache', + '.local/share/NuGet/v3-cache', + '.nuget/packages', + ]; + /** The architecture name used by the dotnet sdk archives. */ private get arch(): string { switch (this.envSvc.arch) { diff --git a/src/cli/tools/erlang/elixir.ts b/src/cli/tools/erlang/elixir.ts index b572c66c8b..4c064e0339 100644 --- a/src/cli/tools/erlang/elixir.ts +++ b/src/cli/tools/erlang/elixir.ts @@ -37,6 +37,12 @@ export class ElixirInstallService extends BaseInstallService { override readonly parent = 'erlang'; override readonly minVersion = '1.4.3'; + /** + * The packages hex downloaded, the hex and rebar archives in `.mix` are + * kept, they are installed by `link`. + */ + override readonly cacheDirs = ['.hex/packages']; + /** * Downloads the elixir archive from GitHub, verified against its * `.sha256sum` since v1.14.0, and extracts it into the versioned tool diff --git a/src/cli/tools/flutter.ts b/src/cli/tools/flutter.ts index ed6b2e3096..fa35ecdd13 100644 --- a/src/cli/tools/flutter.ts +++ b/src/cli/tools/flutter.ts @@ -8,6 +8,7 @@ import { initPubCache, prepareDartHome, preparePubCache, + pubCacheDirs, } from './dart/utils.ts'; @injectable() @@ -72,6 +73,12 @@ export class FlutterInstallService extends BaseInstallService { /** The oldest prebuild. */ override readonly minVersion = '1.22.6'; + /** + * The packages pub downloaded, globally activated packages in the pub cache + * are kept. + */ + override readonly cacheDirs = pubCacheDirs; + /** The architecture name used by the flutter prebuilds. */ private get ghArch(): string { switch (this.envSvc.arch) { diff --git a/src/cli/tools/golang.ts b/src/cli/tools/golang.ts index b3118e708a..6071f07d51 100644 --- a/src/cli/tools/golang.ts +++ b/src/cli/tools/golang.ts @@ -60,6 +60,12 @@ export class GolangInstallService extends BaseInstallService { override readonly name = 'golang'; override readonly minVersion = '1.9.0'; + /** + * The module cache, the build cache is in `~/.cache/go-build`, which is + * cleaned anyway. + */ + override readonly cacheDirs = ['go/pkg/mod']; + /** The architecture name used by the containerbase prebuild. */ private get prebuildArch(): string { return this.envSvc.arch === 'arm64' ? 'aarch64' : 'x86_64'; diff --git a/src/cli/tools/java/index.ts b/src/cli/tools/java/index.ts index a409179a4f..28f2be98f3 100644 --- a/src/cli/tools/java/index.ts +++ b/src/cli/tools/java/index.ts @@ -146,6 +146,17 @@ export class JavaInstallService extends BaseInstallService { /** The oldest Temurin release adoptium serves. */ override readonly minVersion = '8.0.302+8'; + /** + * The maven repository, the gradle caches and the distributions the maven + * and gradle wrappers downloaded. + */ + override readonly cacheDirs = [ + '.gradle/caches', + '.gradle/wrapper/dists', + '.m2/repository', + '.m2/wrapper/dists', + ]; + /** * Downloads the adoptium jdk or jre, verified against its checksum, into the * versioned tool path, adds the revoked intermediate CAs to its blocked list diff --git a/src/cli/tools/java/sbt.ts b/src/cli/tools/java/sbt.ts index 5c0a09d86a..b72b384a55 100644 --- a/src/cli/tools/java/sbt.ts +++ b/src/cli/tools/java/sbt.ts @@ -35,6 +35,9 @@ export class SbtInstallService extends BaseInstallService { override readonly minVersion = '0.13.17'; override readonly parent = 'java'; + /** The sbt launcher's downloaded sbt and scala versions. */ + override readonly cacheDirs = ['.sbt/boot']; + /** * Downloads the sbt archive from GitHub, verified against its `.sha256` * since v1.3.5, extracts it into the versioned tool path and drops the diff --git a/src/cli/tools/node/index.ts b/src/cli/tools/node/index.ts index c1f0dcc829..2e047806ab 100644 --- a/src/cli/tools/node/index.ts +++ b/src/cli/tools/node/index.ts @@ -56,6 +56,17 @@ export class NodeInstallService extends NodeBaseInstallService { readonly name = 'node'; override readonly minVersion = '8.7.0'; + /** + * The npm package cache, the packages `npx` downloaded and the pnpm store, + * `~/.local` links to the cache. The yarn classic cache is in `~/.cache`, + * which is cleaned anyway. + */ + override readonly cacheDirs = [ + '.local/share/pnpm/store', + '.npm/_cacache', + '.npm/_npx', + ]; + /** The architecture name used by the nodejs.org archives. */ private get nodeArch(): string { switch (this.envSvc.arch) { diff --git a/src/cli/tools/python/conan.ts b/src/cli/tools/python/conan.ts index 3c22def729..a816628832 100644 --- a/src/cli/tools/python/conan.ts +++ b/src/cli/tools/python/conan.ts @@ -56,6 +56,9 @@ export class ConanPrepareService extends BasePrepareService { @injectFromHierarchy() export class ConanInstallService extends PipBaseInstallService { override readonly name: string = 'conan'; + + /** The package cache, the profiles in `.conan2/profiles` are kept. */ + override readonly cacheDirs = ['.conan2/p']; } @injectable() diff --git a/src/cli/tools/ruby/cocoapods.ts b/src/cli/tools/ruby/cocoapods.ts index ca51a4f4e7..120896bf0d 100644 --- a/src/cli/tools/ruby/cocoapods.ts +++ b/src/cli/tools/ruby/cocoapods.ts @@ -8,6 +8,12 @@ import { RubyBaseInstallService, RubyGemVersionResolver } from './utils.ts'; export class CocoapodsInstallService extends RubyBaseInstallService { override readonly name: string = 'cocoapods'; + /** + * The pods cocoapods downloaded, also on linux below `~/Library`. The spec + * repos in `.cocoapods/repos` are kept, they can be private ones. + */ + override readonly cacheDirs = ['Library/Caches/CocoaPods']; + /** Checks that `pod --version` runs, also as root. */ override async test(_version: string): Promise { await this._spawn('pod', ['--version', '--allow-root']); diff --git a/src/cli/tools/ruby/index.ts b/src/cli/tools/ruby/index.ts index 1649ce24c4..1de8e70626 100644 --- a/src/cli/tools/ruby/index.ts +++ b/src/cli/tools/ruby/index.ts @@ -9,7 +9,7 @@ import { PrebuildInstallService } from '../utils/prebuild.ts'; const supportedDistros = ['jammy', 'noble', 'resolute']; /** The user home entries redirected into the containerbase cache. */ -const cacheDirs = ['.gem', '.cocoapods', 'Library']; +const linkedDirs = ['.gem', '.cocoapods', 'Library']; @injectable() @injectFromHierarchy() @@ -38,7 +38,7 @@ export class RubyPrepareService extends BasePrepareService { await this.initialize(); - for (const entry of ['.gemrc', ...cacheDirs]) { + for (const entry of ['.gemrc', ...linkedDirs]) { await this.pathSvc.createSymlink( join(this.pathSvc.cachePath, entry), join(this.envSvc.userHome, entry), @@ -68,7 +68,7 @@ export class RubyPrepareService extends BasePrepareService { `, ); - for (const dir of cacheDirs) { + for (const dir of linkedDirs) { await this.pathSvc.createDir(join(this.pathSvc.cachePath, dir)); } } diff --git a/src/cli/tools/rust.ts b/src/cli/tools/rust.ts index 83c7525179..a8dabc8bd0 100644 --- a/src/cli/tools/rust.ts +++ b/src/cli/tools/rust.ts @@ -34,6 +34,9 @@ export class RustInstallService extends BaseInstallService { override readonly name = 'rust'; override readonly minVersion = '1.7.0'; + /** The crates and git dependencies cargo downloaded. */ + override readonly cacheDirs = ['.cargo/git', '.cargo/registry']; + /** The architecture name used by the rust release archives. */ private get rustArch(): string { return this.envSvc.arch === 'arm64' ? 'aarch64' : 'x86_64';