From 60f04e1690786c3988af6eae52972a4f4eeb4650 Mon Sep 17 00:00:00 2001 From: Nitish Agarwal <1592163+nitishagar@users.noreply.github.com> Date: Thu, 2 Jul 2026 08:41:40 +0530 Subject: [PATCH] feat(python): add layers map for separate per-layer dependency sets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `custom.pythonRequirements.layers` — a named map where each entry specifies its own requirements file. For each entry the plugin: 1. Installs the explicit requirements file into a per-layer working directory via a new `installRequirementsForFile` helper in pip.js. 2. Zips the installed packages under a `python/` prefix to `.serverless/pythonRequirements-.zip`. 3. Registers `service.layers[]` with `package.artifact` set so the core layer compiler emits `LambdaLayer` and `LambdaLayerQualifiedArn` without further changes. Functions attach a layer with `Ref: LambdaLayer`. Invariants preserved: - The existing single-layer (`layer: true`) path is untouched. - `layer` and `layers` coexist independently. - The reserved name `pythonRequirements` and collisions with existing service layers are rejected before any install work begins. - Registration is deferred to a post-loop `Object.assign` so a mid-loop failure never leaves a half-registered state. Adds a JSON schema for `custom.pythonRequirements.layers` (via `defineCustomProperties`) and extends the function-less-service hook guard to keep hooks alive for `layers`-only services. Closes #13384 --- docs/sf/providers/aws/guide/python.md | 34 +++ .../serverless/lib/plugins/python/index.js | 60 ++++- .../lib/plugins/python/lib/layer.js | 163 ++++++++++++- .../serverless/lib/plugins/python/lib/pip.js | 86 ++++++- .../unit/lib/plugins/python/layer.test.js | 230 ++++++++++++++++++ .../test/unit/lib/plugins/python/pip.test.js | 182 ++++++++++++++ packages/sf-core/tests/python/test.js | 37 +++ .../layers_multi/requirements/pydantic.txt | 1 + .../tests/layers_multi/requirements/ulid.txt | 1 + .../python/tests/layers_multi/serverless.yml | 20 ++ 10 files changed, 808 insertions(+), 6 deletions(-) create mode 100644 packages/serverless/test/unit/lib/plugins/python/layer.test.js create mode 100644 packages/serverless/test/unit/lib/plugins/python/pip.test.js create mode 100644 packages/sf-core/tests/python/tests/layers_multi/requirements/pydantic.txt create mode 100644 packages/sf-core/tests/python/tests/layers_multi/requirements/ulid.txt create mode 100644 packages/sf-core/tests/python/tests/layers_multi/serverless.yml diff --git a/docs/sf/providers/aws/guide/python.md b/docs/sf/providers/aws/guide/python.md index 5e0c0e0ee8c..db6f9e7fab0 100644 --- a/docs/sf/providers/aws/guide/python.md +++ b/docs/sf/providers/aws/guide/python.md @@ -333,6 +333,40 @@ custom: - '*' ``` +### Multiple Layers + +To split dependencies across several Lambda layers — each built from its own +requirements file — use the `layers` map. The map key is the layer name; each entry +points at its own requirements file. + +```yaml +custom: + pythonRequirements: + layers: + pydantic: + requirementsFile: requirements/pydantic.txt + web: + requirementsFile: requirements/web.txt + description: FastAPI + Pydantic runtime + +functions: + api: + handler: handler.api + layers: + - Ref: PydanticLambdaLayer + - Ref: WebLambdaLayer +``` + +Each layer is packaged independently under the `python/` path. Reference a layer +from a function with `Ref: LambdaLayer` (e.g. a key `pydantic` becomes +`Ref: PydanticLambdaLayer`). `name`, `description`, `compatibleRuntimes`, and the +other standard layer properties may be set per entry; when omitted, +`compatibleRuntimes` defaults to the provider runtime and `name` defaults to +`${service}-${stage}-`. + +`layers` is independent of `layer`: you may use either or both in the same service. +The name `pythonRequirements` is reserved for the single `layer` feature. + ## Omitting Packages You can omit a package from deployment with the `noDeploy` option. Note that diff --git a/packages/serverless/lib/plugins/python/index.js b/packages/serverless/lib/plugins/python/index.js index cc6b27f7f2b..a3f4ff78f14 100644 --- a/packages/serverless/lib/plugins/python/index.js +++ b/packages/serverless/lib/plugins/python/index.js @@ -124,6 +124,12 @@ class ServerlessPythonRequirements { options.layer = {} } } + if ( + options.layers != null && + (typeof options.layers !== 'object' || Array.isArray(options.layers)) + ) { + delete options.layers + } return options } @@ -206,6 +212,54 @@ class ServerlessPythonRequirements { }) } + if (this.serverless.configSchemaHandler?.defineCustomProperties) { + this.serverless.configSchemaHandler.defineCustomProperties({ + properties: { + pythonRequirements: { + type: 'object', + properties: { + layers: { + description: `Named Python requirement layers, each built from its own requirements file. +@remarks Map key becomes the layer name; reference with Ref: LambdaLayer.`, + type: 'object', + additionalProperties: { + type: 'object', + properties: { + requirementsFile: { + description: `Path (relative to service root) to this layer's requirements.txt.`, + type: 'string', + minLength: 1, + }, + name: { type: 'string', minLength: 1, maxLength: 140 }, + description: { type: 'string', maxLength: 256 }, + compatibleRuntimes: { + type: 'array', + items: { $ref: '#/definitions/awsLambdaRuntime' }, + maxItems: 15, + }, + compatibleArchitectures: { + type: 'array', + items: { $ref: '#/definitions/awsLambdaArchitecture' }, + maxItems: 2, + }, + licenseInfo: { type: 'string', maxLength: 512 }, + allowedAccounts: { + type: 'array', + items: { type: 'string' }, + }, + retain: { type: 'boolean' }, + }, + required: ['requirementsFile'], + additionalProperties: false, + }, + }, + }, + additionalProperties: true, + }, + }, + }) + } + if (v3Utils) { this.log = v3Utils.log this.progress = v3Utils.progress @@ -277,8 +331,12 @@ class ServerlessPythonRequirements { const layerConfigured = Boolean( this.serverless.service.custom?.pythonRequirements?.layer, ) + const layersConfigured = Boolean( + this.serverless.service.custom?.pythonRequirements?.layers, + ) const hasPythonLayerOnly = - layerConfigured && providerRuntime?.startsWith('python') + (layerConfigured || layersConfigured) && + providerRuntime?.startsWith('python') return hasPythonFunction || hasPythonAgent || hasPythonLayerOnly } diff --git a/packages/serverless/lib/plugins/python/lib/layer.js b/packages/serverless/lib/plugins/python/lib/layer.js index 19d4a2dfa78..5ccb3135fa4 100644 --- a/packages/serverless/lib/plugins/python/lib/layer.js +++ b/packages/serverless/lib/plugins/python/lib/layer.js @@ -1,8 +1,10 @@ import fse from 'fs-extra' import path from 'path' import JSZip from 'jszip' +import { ServerlessError } from '@serverless/util' import { writeZip, addTree } from './zipTree.js' import { sha256Path, getRequirementsLayerPath } from './shared.js' +import { installRequirementsForFile } from './pip.js' /** * Zip up requirements to be used as layer package. @@ -48,7 +50,7 @@ async function zipRequirements() { if (process.platform === 'win32') { fse.copySync(zipCachePath, targetZipPath) } else { - fse.symlink(zipCachePath, targetZipPath, 'file') + await fse.symlink(zipCachePath, targetZipPath, 'file') } } } @@ -76,12 +78,160 @@ async function createLayers() { return } +/** + * Validates a named layer name against reserved tokens and existing layers. + * @param {string} name + * @param {Object} existingLayers + */ +function validateLayerName(name, existingLayers) { + if (name === 'pythonRequirements') { + throw new ServerlessError( + `Python Requirements: layer name "pythonRequirements" is reserved for the single-layer feature. Choose a different name.`, + 'PYTHON_REQUIREMENTS_LAYER_NAME_RESERVED', + { stack: false }, + ) + } + if ( + existingLayers && + Object.prototype.hasOwnProperty.call(existingLayers, name) + ) { + throw new ServerlessError( + `Python Requirements: layer name "${name}" collides with an already-declared layer. Use a unique name.`, + 'PYTHON_REQUIREMENTS_LAYER_NAME_COLLISION', + { stack: false }, + ) + } +} + +/** + * Zip up a named layer's installed packages under a python/ prefix. + * @param {string} name - layer name + * @param {string} installDir - directory containing installed packages + * @return {Promise} + */ +async function zipNamedLayerRequirements(name, installDir) { + const servicePath = + this.servicePath || + this.serverless?.config?.servicePath || + this.serverless?.serviceDir || + process.cwd() + const requirementsDir = path.join(servicePath, '.serverless') + const layerDir = path.join(requirementsDir, `pythonRequirements-${name}`) + const requirementsTxtPath = path.join(layerDir, 'requirements.txt') + const targetZipPath = path.join( + requirementsDir, + `pythonRequirements-${name}.zip`, + ) + + // A missing or empty (zero-byte) requirements.txt means there are no + // dependencies to package, so emit a clean empty layer zip. pip.js returns + // the layer dir with a zero-byte requirements.txt when the resolved + // requirements are empty, so existence alone is not enough. + if ( + !fse.existsSync(requirementsTxtPath) || + fse.statSync(requirementsTxtPath).size === 0 + ) { + const rootZip = new JSZip() + rootZip.folder('python') + await writeZip(rootZip, targetZipPath) + return + } + + const reqChecksum = sha256Path(requirementsTxtPath) + const zipCachePath = getRequirementsLayerPath( + reqChecksum, + targetZipPath, + this.options, + this.serverless, + ) + + if (fse.existsSync(zipCachePath)) { + if (this.progress && this.log) { + this.log.info( + `Found cached Python Requirements Lambda Layer file for layer ${name}`, + ) + } else { + this.serverless.cli.log( + `Found cached Python Requirements Lambda Layer file for layer ${name}`, + ) + } + } else { + const rootZip = new JSZip() + const folder = rootZip.folder('python') + await addTree(folder, installDir) + await writeZip(rootZip, zipCachePath) + } + + if (zipCachePath !== targetZipPath) { + if (process.platform === 'win32') { + fse.copySync(zipCachePath, targetZipPath) + } else { + await fse.symlink(zipCachePath, targetZipPath, 'file') + } + } +} + +/** + * Install, zip, and register all named layers declared in options.layers. + * Uses a two-pass approach: pre-flight validation then build loop with deferred registration. + * @return {Promise} + */ +async function createNamedLayers() { + if (!this.serverless.service.layers) { + this.serverless.service.layers = {} + } + const existingLayers = this.serverless.service.layers + const entries = Object.entries(this.options.layers) + + for (const [name] of entries) { + validateLayerName(name, existingLayers) + } + + const accumulator = {} + for (const [name, cfg] of entries) { + const installDir = await installRequirementsForFile( + cfg.requirementsFile, + name, + this, + ) + await zipNamedLayerRequirements.call(this, name, installDir) + + const { + requirementsFile: _rf, + name: cfgName, + description, + compatibleRuntimes, + ...restOfUserProps + } = cfg + + accumulator[name] = { + package: { + artifact: path.join('.serverless', `pythonRequirements-${name}.zip`), + }, + name: + cfgName || + `${this.serverless.service.service}-${this.serverless.providers.aws.getStage()}-${name}`, + description: + description || + `Python requirements for layer ${name} generated by serverless-python-requirements.`, + compatibleRuntimes: compatibleRuntimes || [ + this.serverless.service.provider.runtime, + ], + ...restOfUserProps, + } + } + + Object.assign(this.serverless.service.layers, accumulator) +} + /** * Creates a layer from the installed requirements. * @return {Promise} the combined promise for requirements layer. */ async function layerRequirements() { - if (!this.options.layer) return + const hasNamedLayers = + this.options.layers && typeof this.options.layers === 'object' + if (!this.options.layer && !hasNamedLayers) return let layerProgress if (this.progress && this.log) { @@ -93,8 +243,13 @@ async function layerRequirements() { } try { - await zipRequirements.call(this) - await createLayers.call(this) + if (this.options.layer) { + await zipRequirements.call(this) + await createLayers.call(this) + } + if (hasNamedLayers) { + await createNamedLayers.call(this) + } } finally { layerProgress && layerProgress.remove() } diff --git a/packages/serverless/lib/plugins/python/lib/pip.js b/packages/serverless/lib/plugins/python/lib/pip.js index e272679e82e..1c847aa8d68 100644 --- a/packages/serverless/lib/plugins/python/lib/pip.js +++ b/packages/serverless/lib/plugins/python/lib/pip.js @@ -1035,4 +1035,88 @@ async function installAllRequirements() { } } -export { installAllRequirements } +/** + * Install requirements from an explicit requirements file into a per-layer working directory. + * Mirrors installRequirementsIfNeeded but is driven by an explicit file rather than modulePath. + * @param {string} requirementsFile - path (absolute or relative to servicePath) to requirements file + * @param {string} layerName - name of the layer (used to derive the working directory) + * @param {Object} pluginInstance + * @return {Promise} path to the directory containing the installed packages + */ +async function installRequirementsForFile( + requirementsFile, + layerName, + pluginInstance, +) { + const { servicePath, options, serverless } = pluginInstance + const absRequirementsFile = path.isAbsolute(requirementsFile) + ? requirementsFile + : path.resolve(servicePath, requirementsFile) + + if ( + !fse.pathExistsSync(absRequirementsFile) || + !fse.statSync(absRequirementsFile).isFile() + ) { + throw new ServerlessError( + `Python Requirements: layer "${layerName}" requirementsFile "${requirementsFile}" does not exist or is not a file.`, + 'PYTHON_REQUIREMENTS_LAYER_REQUIREMENTS_FILE_INVALID', + { stack: false }, + ) + } + + const layerDir = path.join( + servicePath, + '.serverless', + `pythonRequirements-${layerName}`, + ) + fse.ensureDirSync(layerDir) + const slsReqsTxt = path.join(layerDir, 'requirements.txt') + + generateRequirementsFile(absRequirementsFile, slsReqsTxt, pluginInstance) + + if (!fse.existsSync(slsReqsTxt) || fse.statSync(slsReqsTxt).size === 0) { + return layerDir + } + + const reqChecksum = sha256Path(slsReqsTxt) + const workingReqsFolder = getRequirementsWorkingPath( + reqChecksum, + layerDir, + options, + serverless, + ) + + if (fse.existsSync(workingReqsFolder)) { + if ( + fse.existsSync(path.join(workingReqsFolder, '.completed_requirements')) && + workingReqsFolder.endsWith('_slspyc') + ) { + fse.utimesSync(workingReqsFolder, new Date(), new Date()) + return workingReqsFolder + } + if ( + workingReqsFolder.endsWith('_slspyc') || + workingReqsFolder.endsWith('.requirements') + ) { + fse.removeSync(workingReqsFolder) + } + } + + fse.ensureDirSync(workingReqsFolder) + fse.copySync(slsReqsTxt, path.join(workingReqsFolder, 'requirements.txt')) + await installRequirements(workingReqsFolder, pluginInstance, {}) + if (options.vendor) { + copyVendors(options.vendor, workingReqsFolder, pluginInstance) + } + if (options.useStaticCache) { + fse.closeSync( + fse.openSync( + path.join(workingReqsFolder, '.completed_requirements'), + 'w', + ), + ) + } + return workingReqsFolder +} + +export { installAllRequirements, installRequirementsForFile } diff --git a/packages/serverless/test/unit/lib/plugins/python/layer.test.js b/packages/serverless/test/unit/lib/plugins/python/layer.test.js new file mode 100644 index 00000000000..c179fbff5d7 --- /dev/null +++ b/packages/serverless/test/unit/lib/plugins/python/layer.test.js @@ -0,0 +1,230 @@ +import { jest, describe, it, expect, beforeEach } from '@jest/globals' +import path from 'path' + +jest.unstable_mockModule('@serverless/util', () => ({ + ServerlessError: class ServerlessError extends Error { + constructor(message, code) { + super(message) + this.code = code + } + }, +})) + +jest.unstable_mockModule('fs-extra', () => ({ + default: { + existsSync: jest.fn().mockReturnValue(true), + statSync: jest.fn().mockReturnValue({ size: 1 }), + ensureDirSync: jest.fn(), + copySync: jest.fn(), + symlink: jest.fn(), + }, +})) + +jest.unstable_mockModule('jszip', () => ({ + default: jest.fn().mockImplementation(() => ({ + folder: jest.fn().mockReturnThis(), + })), +})) + +jest.unstable_mockModule( + '../../../../../lib/plugins/python/lib/zipTree.js', + () => ({ + addTree: jest.fn().mockResolvedValue(undefined), + writeZip: jest.fn().mockResolvedValue(undefined), + }), +) + +jest.unstable_mockModule( + '../../../../../lib/plugins/python/lib/shared.js', + () => ({ + sha256Path: jest.fn().mockReturnValue('abc123'), + getRequirementsLayerPath: jest + .fn() + .mockImplementation((_, targetZipPath) => targetZipPath), + }), +) + +jest.unstable_mockModule( + '../../../../../lib/plugins/python/lib/pip.js', + () => ({ + installAllRequirements: jest.fn().mockResolvedValue(undefined), + installRequirementsForFile: jest + .fn() + .mockResolvedValue('/mock/install/dir'), + }), +) + +const { layerRequirements } = + await import('../../../../../lib/plugins/python/lib/layer.js') +const { installRequirementsForFile: installRequirementsForFileMock } = + await import('../../../../../lib/plugins/python/lib/pip.js') +const { ServerlessError } = await import('@serverless/util') + +const SERVICE_PATH = '/mock/service' + +function makePluginInstance({ + layers = undefined, + layer = false, + existingServiceLayers = {}, + runtime = 'python3.13', + stage = 'dev', + service = 'svc', +} = {}) { + return { + servicePath: SERVICE_PATH, + options: { + layer, + layers, + }, + serverless: { + service: { + service, + provider: { runtime }, + layers: existingServiceLayers, + }, + providers: { + aws: { getStage: jest.fn().mockReturnValue(stage) }, + }, + cli: { log: jest.fn() }, + }, + progress: null, + log: null, + } +} + +describe('layerRequirements — named layers', () => { + beforeEach(() => { + jest.clearAllMocks() + installRequirementsForFileMock.mockResolvedValue('/mock/install/dir') + }) + + it('registers one named layer with defaults when layers has a single entry', async () => { + const instance = makePluginInstance({ + layers: { pydantic: { requirementsFile: 'r.txt' } }, + }) + await layerRequirements.call(instance) + + expect(instance.serverless.service.layers.pydantic).toBeDefined() + const layer = instance.serverless.service.layers.pydantic + expect(layer.package.artifact).toBe( + path.join('.serverless', 'pythonRequirements-pydantic.zip'), + ) + expect(layer.name).toBe('svc-dev-pydantic') + expect(layer.compatibleRuntimes).toEqual(['python3.13']) + expect(installRequirementsForFileMock).toHaveBeenCalledTimes(1) + expect(installRequirementsForFileMock).toHaveBeenCalledWith( + 'r.txt', + 'pydantic', + instance, + ) + }) + + it('forwards user overrides for name, description, compatibleRuntimes, allowedAccounts', async () => { + const instance = makePluginInstance({ + layers: { + mylib: { + requirementsFile: 'reqs/mylib.txt', + name: 'custom-layer-name', + description: 'My custom layer', + compatibleRuntimes: ['python3.12'], + allowedAccounts: ['123456789012'], + }, + }, + }) + await layerRequirements.call(instance) + + const layer = instance.serverless.service.layers.mylib + expect(layer.name).toBe('custom-layer-name') + expect(layer.description).toBe('My custom layer') + expect(layer.compatibleRuntimes).toEqual(['python3.12']) + expect(layer.allowedAccounts).toEqual(['123456789012']) + }) + + it('does nothing when layers is an empty object', async () => { + const instance = makePluginInstance({ layers: {} }) + await layerRequirements.call(instance) + + expect(installRequirementsForFileMock).not.toHaveBeenCalled() + expect(instance.serverless.service.layers).toEqual({}) + }) + + it('returns early without error when neither layer nor layers is set', async () => { + const instance = makePluginInstance() + await layerRequirements.call(instance) + + expect(installRequirementsForFileMock).not.toHaveBeenCalled() + }) + + it('throws PYTHON_REQUIREMENTS_LAYER_NAME_RESERVED before any install for reserved name', async () => { + const instance = makePluginInstance({ + layers: { pythonRequirements: { requirementsFile: 'r.txt' } }, + }) + + await expect(layerRequirements.call(instance)).rejects.toMatchObject({ + code: 'PYTHON_REQUIREMENTS_LAYER_NAME_RESERVED', + }) + expect(installRequirementsForFileMock).not.toHaveBeenCalled() + }) + + it('throws PYTHON_REQUIREMENTS_LAYER_NAME_COLLISION before any install when name collides', async () => { + const instance = makePluginInstance({ + layers: { existing: { requirementsFile: 'r.txt' } }, + existingServiceLayers: { existing: {} }, + }) + + await expect(layerRequirements.call(instance)).rejects.toMatchObject({ + code: 'PYTHON_REQUIREMENTS_LAYER_NAME_COLLISION', + }) + expect(installRequirementsForFileMock).not.toHaveBeenCalled() + }) + + it('does not register any layer when mid-loop install fails (INV-8)', async () => { + installRequirementsForFileMock + .mockResolvedValueOnce('/mock/install/first') + .mockRejectedValueOnce(new Error('install failed')) + + const instance = makePluginInstance({ + layers: { + first: { requirementsFile: 'first.txt' }, + second: { requirementsFile: 'second.txt' }, + }, + }) + + await expect(layerRequirements.call(instance)).rejects.toThrow( + 'install failed', + ) + expect(instance.serverless.service.layers.first).toBeUndefined() + expect(instance.serverless.service.layers.second).toBeUndefined() + }) + + it('leaves single-layer pythonRequirements intact when named layer install fails', async () => { + installRequirementsForFileMock.mockRejectedValue(new Error('fail')) + + const fse = (await import('fs-extra')).default + fse.existsSync.mockReturnValue(true) + + const instance = makePluginInstance({ + layer: {}, + layers: { mylib: { requirementsFile: 'r.txt' } }, + existingServiceLayers: {}, + }) + + await expect(layerRequirements.call(instance)).rejects.toThrow('fail') + expect(instance.serverless.service.layers.pythonRequirements).toBeDefined() + expect(instance.serverless.service.layers.mylib).toBeUndefined() + }) + + it('registers only pythonRequirements when layer is set and layers is absent', async () => { + const fse = (await import('fs-extra')).default + fse.existsSync.mockReturnValue(true) + + const instance = makePluginInstance({ + layer: {}, + layers: undefined, + }) + await layerRequirements.call(instance) + + expect(instance.serverless.service.layers.pythonRequirements).toBeDefined() + expect(installRequirementsForFileMock).not.toHaveBeenCalled() + }) +}) diff --git a/packages/serverless/test/unit/lib/plugins/python/pip.test.js b/packages/serverless/test/unit/lib/plugins/python/pip.test.js new file mode 100644 index 00000000000..ba2981d099f --- /dev/null +++ b/packages/serverless/test/unit/lib/plugins/python/pip.test.js @@ -0,0 +1,182 @@ +import { jest, describe, it, expect, beforeEach } from '@jest/globals' +import path from 'path' + +jest.unstable_mockModule('@serverless/util', () => ({ + ServerlessError: class ServerlessError extends Error { + constructor(message, code) { + super(message) + this.code = code + } + }, +})) + +const fseMock = { + pathExistsSync: jest.fn(), + statSync: jest.fn(), + ensureDirSync: jest.fn(), + existsSync: jest.fn(), + copySync: jest.fn(), + utimesSync: jest.fn(), + removeSync: jest.fn(), + openSync: jest.fn().mockReturnValue(3), + closeSync: jest.fn(), + readFileSync: jest.fn().mockReturnValue('pydantic==1.0.0\n'), + writeFileSync: jest.fn(), +} +jest.unstable_mockModule('fs-extra', () => ({ default: fseMock })) + +jest.unstable_mockModule('child-process-ext/spawn.js', () => ({ + default: jest.fn().mockResolvedValue({ + stdoutBuffer: Buffer.from(''), + stderrBuffer: Buffer.from(''), + }), +})) + +jest.unstable_mockModule( + '../../../../../lib/plugins/python/lib/shared.js', + () => ({ + sha256Path: jest.fn().mockReturnValue('abc123'), + getRequirementsWorkingPath: jest + .fn() + .mockImplementation((checksum, dir) => + path.join(dir, `${checksum}_slspyc`), + ), + checkForAndDeleteMaxCacheVersions: jest.fn(), + getUserCachePath: jest.fn().mockReturnValue('/mock/cache'), + }), +) + +jest.unstable_mockModule( + '../../../../../lib/plugins/python/lib/poetry.js', + () => ({ + isPoetryProject: jest.fn().mockReturnValue(false), + pyprojectTomlToRequirements: jest.fn().mockResolvedValue(undefined), + }), +) + +jest.unstable_mockModule('../../../../../lib/plugins/python/lib/uv.js', () => ({ + getUvVersion: jest.fn().mockResolvedValue('1.0.0'), + uvToRequirements: jest.fn().mockResolvedValue(undefined), +})) + +jest.unstable_mockModule( + '../../../../../lib/plugins/python/lib/docker.js', + () => ({ + buildImage: jest.fn(), + getBindPath: jest.fn(), + getDockerUid: jest.fn(), + }), +) + +jest.unstable_mockModule( + '../../../../../lib/plugins/python/lib/slim.js', + () => ({ + deleteFiles: jest.fn(), + getStripCommand: jest.fn(), + getStripMode: jest.fn().mockReturnValue(false), + }), +) + +const { installRequirementsForFile } = + await import('../../../../../lib/plugins/python/lib/pip.js') +const { ServerlessError } = await import('@serverless/util') + +const SERVICE_PATH = '/mock/service' + +function makePluginInstance(options = {}) { + return { + servicePath: SERVICE_PATH, + options: { + slim: false, + slimPatterns: false, + zip: false, + layer: false, + useStaticCache: true, + useDownloadCache: false, + usePoetry: false, + usePipenv: false, + useUv: false, + dockerizePip: false, + pythonBin: 'python', + vendor: '', + noDeploy: [], + pipCmdExtraArgs: [], + installer: null, + ...options, + }, + serverless: { + service: { provider: { runtime: 'python3.13' } }, + cli: { log: jest.fn() }, + config: { servicePath: SERVICE_PATH }, + }, + log: null, + progress: null, + } +} + +describe('installRequirementsForFile', () => { + beforeEach(() => { + jest.clearAllMocks() + fseMock.readFileSync.mockReturnValue('pydantic==1.0.0\n') + fseMock.writeFileSync.mockImplementation(() => {}) + fseMock.ensureDirSync.mockImplementation(() => {}) + fseMock.copySync.mockImplementation(() => {}) + fseMock.utimesSync.mockImplementation(() => {}) + }) + + it('(a) returns the working folder on a cache hit', async () => { + const reqFile = 'requirements/pydantic.txt' + const absReqFile = path.resolve(SERVICE_PATH, reqFile) + const layerDir = path.join( + SERVICE_PATH, + '.serverless', + 'pythonRequirements-pydantic', + ) + const expectedFolder = path.join(layerDir, 'abc123_slspyc') + + fseMock.pathExistsSync.mockReturnValue(true) + fseMock.statSync.mockImplementation((p) => { + if (p === absReqFile) return { isFile: () => true } + return { size: 10, isFile: () => true } + }) + fseMock.existsSync.mockImplementation((p) => { + if (p === expectedFolder) return true + if (p === path.join(expectedFolder, '.completed_requirements')) + return true + return true + }) + + const result = await installRequirementsForFile( + reqFile, + 'pydantic', + makePluginInstance(), + ) + + expect(result).toBe(expectedFolder) + }) + + it('(b) throws PYTHON_REQUIREMENTS_LAYER_REQUIREMENTS_FILE_INVALID when file does not exist', async () => { + fseMock.pathExistsSync.mockReturnValue(false) + + await expect( + installRequirementsForFile( + 'missing.txt', + 'pydantic', + makePluginInstance(), + ), + ).rejects.toMatchObject({ + code: 'PYTHON_REQUIREMENTS_LAYER_REQUIREMENTS_FILE_INVALID', + }) + }) + + it('(c) throws PYTHON_REQUIREMENTS_LAYER_REQUIREMENTS_FILE_INVALID when path is a directory', async () => { + fseMock.pathExistsSync.mockReturnValue(true) + fseMock.statSync.mockReturnValue({ isFile: () => false }) + + await expect( + installRequirementsForFile('somedir', 'pydantic', makePluginInstance()), + ).rejects.toMatchObject({ + code: 'PYTHON_REQUIREMENTS_LAYER_REQUIREMENTS_FILE_INVALID', + }) + }) +}) diff --git a/packages/sf-core/tests/python/test.js b/packages/sf-core/tests/python/test.js index f3b5110d840..5f3c873b116 100644 --- a/packages/sf-core/tests/python/test.js +++ b/packages/sf-core/tests/python/test.js @@ -315,6 +315,43 @@ test('layer-only service (no functions) still produces lambda layer', async (t) t.end() }) +test('multiple named requirement layers each built from their own requirements file', async (t) => { + process.chdir('tests/layers_multi') + sls(['package'], { env: {} }) + const cfn = JSON.parse( + await readFile( + '.serverless/cloudformation-template-update-stack.json', + 'utf8', + ), + ) + t.ok( + cfn.Resources?.PydanticLambdaLayer, + 'PydanticLambdaLayer resource exists', + ) + t.ok(cfn.Resources?.UlidLambdaLayer, 'UlidLambdaLayer resource exists') + t.ok( + cfn.Outputs?.PydanticLambdaLayerQualifiedArn, + 'pydantic qualified ARN output exists', + ) + t.ok( + cfn.Outputs?.UlidLambdaLayerQualifiedArn, + 'ulid qualified ARN output exists', + ) + const pydZip = await listZipFiles( + '.serverless/pythonRequirements-pydantic.zip', + ) + t.true( + pydZip.some((p) => p.startsWith('python/pydantic')), + 'pydantic packaged under python/', + ) + const ulidZip = await listZipFiles('.serverless/pythonRequirements-ulid.zip') + t.true( + ulidZip.some((p) => p.startsWith('python/ulid')), + 'ulid packaged under python/', + ) + t.end() +}) + test('py3.13 packages have the same hash', async (t) => { process.chdir('tests/base') sls(['package'], { env: {} }) diff --git a/packages/sf-core/tests/python/tests/layers_multi/requirements/pydantic.txt b/packages/sf-core/tests/python/tests/layers_multi/requirements/pydantic.txt new file mode 100644 index 00000000000..572b352f30e --- /dev/null +++ b/packages/sf-core/tests/python/tests/layers_multi/requirements/pydantic.txt @@ -0,0 +1 @@ +pydantic diff --git a/packages/sf-core/tests/python/tests/layers_multi/requirements/ulid.txt b/packages/sf-core/tests/python/tests/layers_multi/requirements/ulid.txt new file mode 100644 index 00000000000..5a6022a60d9 --- /dev/null +++ b/packages/sf-core/tests/python/tests/layers_multi/requirements/ulid.txt @@ -0,0 +1 @@ +python-ulid diff --git a/packages/sf-core/tests/python/tests/layers_multi/serverless.yml b/packages/sf-core/tests/python/tests/layers_multi/serverless.yml new file mode 100644 index 00000000000..bb9579aeeda --- /dev/null +++ b/packages/sf-core/tests/python/tests/layers_multi/serverless.yml @@ -0,0 +1,20 @@ +service: sls-py-req-layers-multi + +provider: + name: aws + runtime: python3.13 + +custom: + pythonRequirements: + layers: + pydantic: + requirementsFile: requirements/pydantic.txt + ulid: + requirementsFile: requirements/ulid.txt + +functions: + api: + handler: handler.api + layers: + - Ref: PydanticLambdaLayer + - Ref: UlidLambdaLayer