diff --git a/docs/tools.md b/docs/tools.md index 1a3ddf3089..2f8b6df099 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -567,7 +567,7 @@ It fails right away, without running any step, when the tools can't be loaded, e Removes the known tool caches from `/tmp/containerbase/cache`. -These are the cache folders of every supported tool, whether it is installed or not, eg. the npm cache in `.npm/_cacache`, the pnpm store in `.local/share/pnpm/store`, the maven repository in `.m2/repository` or the go module cache in `go/pkg/mod`. +These are the cache folders of every supported tool, whether it is installed or not, eg. the npm cache in `.npm/_cacache`, the pnpm store in `.local/share/pnpm/store`, the yarn caches in `.yarn/berry/cache` and `.yarn/zpm/cache`, the maven repository in `.m2/repository` or the go module cache in `go/pkg/mod`. The contents of the generic `.cache` folder and the virtualenv cache in `.local/share/virtualenv` are removed too. Config files, like `.npmrc`, `.m2/settings.xml` or the conan profiles, are kept. When a tool's cache folder or `.cache` itself is a link, eg. to a volume, its target is emptied and the link kept, a link to a missing target is skipped. diff --git a/src/cli/services/path.service.spec.ts b/src/cli/services/path.service.spec.ts index 2c36f33123..545aa01879 100644 --- a/src/cli/services/path.service.spec.ts +++ b/src/cli/services/path.service.spec.ts @@ -72,6 +72,43 @@ describe('cli/services/path.service', () => { true, ); expect(await pathExists(rootPath('tmp/containerbase'), 'dir')).toBe(true); + expect(await pathExists(join(pathSvc.cachePath, '.npm'), 'dir')).toBe(true); + expect(await pathExists(join(pathSvc.cachePath, '.yarn'), 'dir')).toBe( + true, + ); + }); + + test('ensureBasePaths keeps a linked npm cache folder', async () => { + const npm = join(pathSvc.cachePath, '.npm'); + await ensurePaths(['var/cache/npm-volume', 'tmp/containerbase/cache']); + await deleteAsync(npm, { force: true }); + await fs.symlink(rootPath('var/cache/npm-volume'), npm); + + try { + await pathSvc.ensureBasePaths(); + expect(await fs.readlink(npm)).toBe(rootPath('var/cache/npm-volume')); + } finally { + await fs.rm(npm); + } + }); + + test('ensureCacheDir skips a link to a missing folder', async () => { + const yarn = join(pathSvc.cachePath, '.yarn'); + await ensurePaths('tmp/containerbase/cache'); + await deleteAsync(yarn, { force: true }); + await fs.symlink(rootPath('var/cache/missing'), yarn); + + try { + await expect(pathSvc.ensureCacheDir('.yarn')).resolves.toBeUndefined(); + expect(await fs.readlink(yarn)).toBe(rootPath('var/cache/missing')); + expect(await pathExists(rootPath('var/cache/missing'))).toBe(false); + expect(logger.debug).toHaveBeenCalledWith( + { path: yarn }, + 'cache link target missing', + ); + } finally { + await fs.rm(yarn); + } }); describe('with CONTAINERBASE_TMP_DIR', () => { diff --git a/src/cli/services/path.service.ts b/src/cli/services/path.service.ts index 8945895708..82a53f2062 100644 --- a/src/cli/services/path.service.ts +++ b/src/cli/services/path.service.ts @@ -392,6 +392,26 @@ export class PathService { await this.createDir(join(this.cachePath, '.cache')); await this.createDir(join(this.cachePath, '.config')); await this.createDir(join(this.cachePath, '.local', 'share')); + // targets of the `~/.npm` and `~/.yarn` links, also used without node's init + await this.ensureCacheDir('.npm'); + await this.ensureCacheDir('.yarn'); + } + + /** + * Creates a folder in the cache path, eg. the target of a home folder link. + * A link to a folder, eg. a volume, is kept, a link to a missing target is + * skipped. + */ + async ensureCacheDir(name: string): Promise { + const path = join(this.cachePath, name); + if (await pathExists(path, 'dir')) { + return; + } + if ((await fs.lstat(path).catch(() => null))?.isSymbolicLink()) { + logger.debug({ path }, 'cache link target missing'); + return; + } + await this.createDir(path); } /** diff --git a/src/cli/tools/node/index.spec.ts b/src/cli/tools/node/index.spec.ts index d9ee0f69f6..11fd0ef3fb 100644 --- a/src/cli/tools/node/index.spec.ts +++ b/src/cli/tools/node/index.spec.ts @@ -417,6 +417,15 @@ describe('cli/tools/node/index', () => { expect(await fs.readlink(join(envSvc.userHome, '.npm'))).toBe( join(pathSvc.cachePath, '.npm'), ); + expect((await fs.stat(join(envSvc.userHome, '.npm'))).isDirectory()).toBe( + true, + ); + expect(await fs.readlink(join(envSvc.userHome, '.yarn'))).toBe( + join(pathSvc.cachePath, '.yarn'), + ); + expect( + (await fs.stat(join(envSvc.userHome, '.yarn'))).isDirectory(), + ).toBe(true); // the tool env is only written once await expect(svc.initialize()).resolves.toBeUndefined(); diff --git a/src/cli/tools/node/index.ts b/src/cli/tools/node/index.ts index 2e047806ab..e877d834e0 100644 --- a/src/cli/tools/node/index.ts +++ b/src/cli/tools/node/index.ts @@ -6,7 +6,6 @@ import { BasePrepareService } from '../../prepare-tool/base-prepare.service.ts'; import { fileContent, parse } from '../../utils/index.ts'; import { NodeBaseInstallService, - prepareNpmCache, prepareNpmrc, prepareSymlinks, } from './utils.ts'; @@ -16,18 +15,20 @@ import { export class NodePrepareService extends BasePrepareService { override name = 'node'; - /** Initializes the cache and links the user's npm folders to it. */ + /** Initializes the cache and links the user's npm and yarn folders to it. */ override async prepare(): Promise { await this.initialize(); await prepareSymlinks(this.envSvc, this.pathSvc); } /** - * Creates the npm cache and `.npmrc`, and exports the node env with update - * notices and funding messages turned off and the openssl ca store used. + * Creates the npm and yarn caches and `.npmrc`, and exports the node env + * with update notices and funding messages turned off and the openssl ca + * store used. */ override async initialize(): Promise { - await prepareNpmCache(this.pathSvc); + await this.pathSvc.ensureCacheDir('.npm'); + await this.pathSvc.ensureCacheDir('.yarn'); await prepareNpmrc(this.pathSvc); if (!(await this.pathSvc.toolEnvExists(this.name))) { @@ -57,14 +58,17 @@ export class NodeInstallService extends NodeBaseInstallService { override readonly minVersion = '8.7.0'; /** - * The npm package cache, the packages `npx` downloaded and the pnpm store, - * `~/.local` links to the cache. The yarn classic cache is in `~/.cache`, + * The npm package cache, the packages `npx` downloaded, the pnpm store, + * `~/.local` links to the cache, and the global caches of yarn 2 to 5 + * (`berry`) and yarn 6 (`zpm`). The yarn classic cache is in `~/.cache`, * which is cleaned anyway. */ override readonly cacheDirs = [ '.local/share/pnpm/store', '.npm/_cacache', '.npm/_npx', + '.yarn/berry/cache', + '.yarn/zpm/cache', ]; /** The architecture name used by the nodejs.org archives. */ diff --git a/src/cli/tools/node/utils.spec.ts b/src/cli/tools/node/utils.spec.ts index 21aae2c19f..0387665118 100644 --- a/src/cli/tools/node/utils.spec.ts +++ b/src/cli/tools/node/utils.spec.ts @@ -12,7 +12,6 @@ import { logger } from '../../utils/index.ts'; import { NpmBaseInstallService, prepareGlobalConfig, - prepareNpmCache, prepareNpmrc, prepareSymlinks, prepareUserConfig, @@ -386,15 +385,9 @@ describe('cli/tools/node/utils', () => { }); describe('helpers', () => { - test('prepareNpmCache', async () => { - await expect(prepareNpmCache(pathSvc)).resolves.toBeUndefined(); - expect( - (await fs.stat(join(pathSvc.cachePath, '.npm'))).isDirectory(), - ).toBe(true); - }); - test('prepareNpmrc', async () => { const npmrc = join(pathSvc.cachePath, '.npmrc'); + await ensurePaths('tmp/containerbase/cache'); await expect(prepareNpmrc(pathSvc)).resolves.toBeUndefined(); expect(await fs.readFile(npmrc, 'utf8')).toBe(''); @@ -416,6 +409,9 @@ describe('cli/tools/node/utils', () => { expect(await fs.readlink(join(envSvc.userHome, '.npmrc'))).toBe( join(pathSvc.cachePath, '.npmrc'), ); + expect(await fs.readlink(join(envSvc.userHome, '.yarn'))).toBe( + join(pathSvc.cachePath, '.yarn'), + ); // second run is a no-op await expect(prepareSymlinks(envSvc, pathSvc)).resolves.toBeUndefined(); diff --git a/src/cli/tools/node/utils.ts b/src/cli/tools/node/utils.ts index 58fd75da30..ebf7bb2756 100644 --- a/src/cli/tools/node/utils.ts +++ b/src/cli/tools/node/utils.ts @@ -302,12 +302,6 @@ async function readPackageJson(path: string): Promise { return JSON.parse(data); } -/** Creates the `.npm` folder in the containerbase cache. */ -export async function prepareNpmCache(pathSvc: PathService): Promise { - const path = join(pathSvc.cachePath, '.npm'); - await pathSvc.createDir(path); -} - /** Creates an empty `.npmrc` in the containerbase cache, unless there is one. */ export async function prepareNpmrc(pathSvc: PathService): Promise { const path = join(pathSvc.cachePath, '.npmrc'); @@ -317,8 +311,8 @@ export async function prepareNpmrc(pathSvc: PathService): Promise { } /** - * Links the user's `.npm` folder and `.npmrc` to the cache, keeping any - * existing link. + * Links the user's `.npm` and `.yarn` folders and `.npmrc` to the cache, + * keeping any existing link. */ export async function prepareSymlinks( envSvc: EnvService, @@ -332,4 +326,8 @@ export async function prepareSymlinks( join(pathSvc.cachePath, '.npmrc'), join(envSvc.userHome, '.npmrc'), ); + await pathSvc.createSymlink( + join(pathSvc.cachePath, '.yarn'), + join(envSvc.userHome, '.yarn'), + ); } diff --git a/test/node/Dockerfile b/test/node/Dockerfile index 6066901dcc..b56d354f9a 100644 --- a/test/node/Dockerfile +++ b/test/node/Dockerfile @@ -636,6 +636,7 @@ RUN set -ex; \ containerbase-cli init tool node; \ [[ "$(readlink /tmp/containerbase)" == /var/cache/containerbase ]]; \ [[ -d /var/cache/containerbase/cache/.npm ]]; \ + [[ -d /var/cache/containerbase/cache/.yarn ]]; \ [[ -f /var/cache/containerbase/cache/.npmrc ]]; \ [[ -d /var/cache/containerbase/cache/.cache ]]; \ rm /var/cache/containerbase/tool.init.d/node @@ -646,7 +647,8 @@ RUN set -ex; \ docker-entrypoint.sh true; \ [[ "$(readlink /tmp/containerbase)" == /var/cache/containerbase ]]; \ [[ ! -e /var/cache/containerbase/tool.init.d/node ]]; \ - [[ -d $USER_HOME/.npm ]] + [[ -d $USER_HOME/.npm ]]; \ + [[ -d $USER_HOME/.yarn ]] # the fast path also takes a value with a trailing slash and keeps it RUN set -ex; \ @@ -683,7 +685,8 @@ RUN set -ex; \ docker-entrypoint.sh true; \ [[ "$(readlink /tmp/containerbase)" == /var/cache/containerbase ]]; \ [[ -d /var/cache/containerbase/tool.init.d ]]; \ - [[ -d $USER_HOME/.npm ]] + [[ -d $USER_HOME/.npm ]]; \ + [[ -d $USER_HOME/.yarn ]] # a boot replaces a link to another folder RUN set -ex; \ @@ -851,6 +854,62 @@ RUN set -ex; \ node --version; \ pnpm --version | grep "^${PNPM_VERSION}$" +# renovate: datasource=npm +RUN install-tool yarn 1.22.22 + +# yarn classic still works with the linked `~/.yarn`, global installs use the npm prefix +RUN set -ex; \ + cd a; \ + yarn install; \ + [[ -d node_modules/semver ]]; \ + [[ "$(yarn global bin)" == /usr/local/bin ]]; \ + yarn global add semver@7.3.2; \ + [[ "$(command -v semver)" == /usr/local/bin/semver ]] + +# renovate: datasource=npm packageName=@yarnpkg/cli-dist +ARG YARN_VERSION=4.18.1 +RUN install-tool yarn "${YARN_VERSION}" + +# the yarn berry global cache is kept in the containerbase cache +RUN set -ex; \ + cd b; \ + yarn install; \ + [[ -L "${USER_HOME}/.yarn" ]]; \ + [[ -n "$(ls -A /tmp/containerbase/cache/.yarn/berry/cache)" ]]; \ + containerbase-cli cleanup cache; \ + [[ ! -e /tmp/containerbase/cache/.yarn/berry/cache ]]; \ + [[ -d /tmp/containerbase/cache/.yarn/berry ]]; \ + yarn install; \ + [[ -n "$(ls -A /tmp/containerbase/cache/.yarn/berry/cache)" ]]; \ + yarn --version | grep "^${YARN_VERSION}$" + +# renovate: datasource=npm packageName=@yarnpkg/yarn-x86_64-unknown-linux-musl +ARG YARN6_VERSION=6.0.0-rc.19 +RUN install-tool yarn "${YARN6_VERSION}" + +# yarn 6 keeps its global cache in `~/.yarn/zpm/cache` +RUN set -ex; \ + mkdir /tmp/yarn6; \ + cp b/package.json /tmp/yarn6/; \ + cd /tmp/yarn6; \ + yarn install; \ + [[ -n "$(ls -A /tmp/containerbase/cache/.yarn/zpm/cache)" ]]; \ + containerbase-cli cleanup cache; \ + [[ ! -e /tmp/containerbase/cache/.yarn/zpm/cache ]]; \ + [[ -d /tmp/containerbase/cache/.yarn/zpm ]]; \ + yarn --version | grep "^${YARN6_VERSION}$" + +USER root +RUN rm -rf /tmp/containerbase +USER 12021 + +# without the entrypoint, the yarn wrapper's init recreates the emptied temp folder with the yarn folder +RUN set -ex; \ + cd /tmp/yarn6; \ + yarn install; \ + [[ -d "${USER_HOME}/.yarn" ]]; \ + [[ -n "$(ls -A /tmp/containerbase/cache/.yarn/zpm/cache)" ]] + #-------------------------------------- # test: vp #--------------------------------------