Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions docs/sf/providers/aws/guide/python.md
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,40 @@ custom:
- '*'
```

### Multiple Layers

To split dependencies across several Lambda layers — each built from its own
requirements file — use the `layers` map. The map key is the layer name; each entry
points at its own requirements file.

```yaml
custom:
pythonRequirements:
layers:
pydantic:
requirementsFile: requirements/pydantic.txt
web:
requirementsFile: requirements/web.txt
description: FastAPI + Pydantic runtime

functions:
api:
handler: handler.api
layers:
- Ref: PydanticLambdaLayer
- Ref: WebLambdaLayer
```

Each layer is packaged independently under the `python/` path. Reference a layer
from a function with `Ref: <Name>LambdaLayer` (e.g. a key `pydantic` becomes
`Ref: PydanticLambdaLayer`). `name`, `description`, `compatibleRuntimes`, and the
other standard layer properties may be set per entry; when omitted,
`compatibleRuntimes` defaults to the provider runtime and `name` defaults to
`${service}-${stage}-<name>`.

`layers` is independent of `layer`: you may use either or both in the same service.
The name `pythonRequirements` is reserved for the single `layer` feature.

## Omitting Packages

You can omit a package from deployment with the `noDeploy` option. Note that
Expand Down
60 changes: 59 additions & 1 deletion packages/serverless/lib/plugins/python/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,12 @@ class ServerlessPythonRequirements {
options.layer = {}
}
}
if (
options.layers != null &&
(typeof options.layers !== 'object' || Array.isArray(options.layers))
) {
delete options.layers
}
return options
}

Expand Down Expand Up @@ -206,6 +212,54 @@ class ServerlessPythonRequirements {
})
}

if (this.serverless.configSchemaHandler?.defineCustomProperties) {
this.serverless.configSchemaHandler.defineCustomProperties({
properties: {
pythonRequirements: {
type: 'object',
properties: {
layers: {
description: `Named Python requirement layers, each built from its own requirements file.
@remarks Map key becomes the layer name; reference with Ref: <Name>LambdaLayer.`,
type: 'object',
additionalProperties: {
type: 'object',
properties: {
requirementsFile: {
description: `Path (relative to service root) to this layer's requirements.txt.`,
type: 'string',
minLength: 1,
},
name: { type: 'string', minLength: 1, maxLength: 140 },
description: { type: 'string', maxLength: 256 },
compatibleRuntimes: {
type: 'array',
items: { $ref: '#/definitions/awsLambdaRuntime' },
maxItems: 15,
},
compatibleArchitectures: {
type: 'array',
items: { $ref: '#/definitions/awsLambdaArchitecture' },
maxItems: 2,
},
licenseInfo: { type: 'string', maxLength: 512 },
allowedAccounts: {
type: 'array',
items: { type: 'string' },
},
retain: { type: 'boolean' },
},
required: ['requirementsFile'],
additionalProperties: false,
},
},
},
additionalProperties: true,
},
},
})
}

if (v3Utils) {
this.log = v3Utils.log
this.progress = v3Utils.progress
Expand Down Expand Up @@ -277,8 +331,12 @@ class ServerlessPythonRequirements {
const layerConfigured = Boolean(
this.serverless.service.custom?.pythonRequirements?.layer,
)
const layersConfigured = Boolean(
this.serverless.service.custom?.pythonRequirements?.layers,
)
const hasPythonLayerOnly =
layerConfigured && providerRuntime?.startsWith('python')
(layerConfigured || layersConfigured) &&
providerRuntime?.startsWith('python')

return hasPythonFunction || hasPythonAgent || hasPythonLayerOnly
}
Expand Down
163 changes: 159 additions & 4 deletions packages/serverless/lib/plugins/python/lib/layer.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import fse from 'fs-extra'
import path from 'path'
import JSZip from 'jszip'
import { ServerlessError } from '@serverless/util'
import { writeZip, addTree } from './zipTree.js'
import { sha256Path, getRequirementsLayerPath } from './shared.js'
import { installRequirementsForFile } from './pip.js'

/**
* Zip up requirements to be used as layer package.
Expand Down Expand Up @@ -48,7 +50,7 @@ async function zipRequirements() {
if (process.platform === 'win32') {
fse.copySync(zipCachePath, targetZipPath)
} else {
fse.symlink(zipCachePath, targetZipPath, 'file')
await fse.symlink(zipCachePath, targetZipPath, 'file')
}
}
}
Expand Down Expand Up @@ -76,12 +78,160 @@ async function createLayers() {
return
}

/**
* Validates a named layer name against reserved tokens and existing layers.
* @param {string} name
* @param {Object} existingLayers
*/
function validateLayerName(name, existingLayers) {
if (name === 'pythonRequirements') {
throw new ServerlessError(
`Python Requirements: layer name "pythonRequirements" is reserved for the single-layer feature. Choose a different name.`,
'PYTHON_REQUIREMENTS_LAYER_NAME_RESERVED',
{ stack: false },
)
}
if (
existingLayers &&
Object.prototype.hasOwnProperty.call(existingLayers, name)
) {
throw new ServerlessError(
`Python Requirements: layer name "${name}" collides with an already-declared layer. Use a unique name.`,
'PYTHON_REQUIREMENTS_LAYER_NAME_COLLISION',
{ stack: false },
)
}
}

/**
* Zip up a named layer's installed packages under a python/ prefix.
* @param {string} name - layer name
* @param {string} installDir - directory containing installed packages
* @return {Promise}
*/
async function zipNamedLayerRequirements(name, installDir) {
const servicePath =
this.servicePath ||
this.serverless?.config?.servicePath ||
this.serverless?.serviceDir ||
process.cwd()
const requirementsDir = path.join(servicePath, '.serverless')
const layerDir = path.join(requirementsDir, `pythonRequirements-${name}`)
const requirementsTxtPath = path.join(layerDir, 'requirements.txt')
const targetZipPath = path.join(
requirementsDir,
`pythonRequirements-${name}.zip`,
)

// A missing or empty (zero-byte) requirements.txt means there are no
// dependencies to package, so emit a clean empty layer zip. pip.js returns
// the layer dir with a zero-byte requirements.txt when the resolved
// requirements are empty, so existence alone is not enough.
if (
!fse.existsSync(requirementsTxtPath) ||
fse.statSync(requirementsTxtPath).size === 0
) {
const rootZip = new JSZip()
rootZip.folder('python')
await writeZip(rootZip, targetZipPath)
return
}

const reqChecksum = sha256Path(requirementsTxtPath)
const zipCachePath = getRequirementsLayerPath(
reqChecksum,
targetZipPath,
this.options,
this.serverless,
)

if (fse.existsSync(zipCachePath)) {
if (this.progress && this.log) {
this.log.info(
`Found cached Python Requirements Lambda Layer file for layer ${name}`,
)
} else {
this.serverless.cli.log(
`Found cached Python Requirements Lambda Layer file for layer ${name}`,
)
}
} else {
const rootZip = new JSZip()
const folder = rootZip.folder('python')
await addTree(folder, installDir)
await writeZip(rootZip, zipCachePath)
}

if (zipCachePath !== targetZipPath) {
if (process.platform === 'win32') {
fse.copySync(zipCachePath, targetZipPath)
} else {
await fse.symlink(zipCachePath, targetZipPath, 'file')
}
}
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

/**
* Install, zip, and register all named layers declared in options.layers.
* Uses a two-pass approach: pre-flight validation then build loop with deferred registration.
* @return {Promise}
*/
async function createNamedLayers() {
if (!this.serverless.service.layers) {
this.serverless.service.layers = {}
}
const existingLayers = this.serverless.service.layers
const entries = Object.entries(this.options.layers)

for (const [name] of entries) {
validateLayerName(name, existingLayers)
}

const accumulator = {}
for (const [name, cfg] of entries) {
const installDir = await installRequirementsForFile(
cfg.requirementsFile,
name,
this,
)
await zipNamedLayerRequirements.call(this, name, installDir)

const {
requirementsFile: _rf,
name: cfgName,
description,
compatibleRuntimes,
...restOfUserProps
} = cfg

accumulator[name] = {
package: {
artifact: path.join('.serverless', `pythonRequirements-${name}.zip`),
},
name:
cfgName ||
`${this.serverless.service.service}-${this.serverless.providers.aws.getStage()}-${name}`,
description:
description ||
`Python requirements for layer ${name} generated by serverless-python-requirements.`,
compatibleRuntimes: compatibleRuntimes || [
this.serverless.service.provider.runtime,
],
...restOfUserProps,
}
}

Object.assign(this.serverless.service.layers, accumulator)
}

/**
* Creates a layer from the installed requirements.
* @return {Promise} the combined promise for requirements layer.
*/
async function layerRequirements() {
if (!this.options.layer) return
const hasNamedLayers =
this.options.layers && typeof this.options.layers === 'object'
if (!this.options.layer && !hasNamedLayers) return

let layerProgress
if (this.progress && this.log) {
Expand All @@ -93,8 +243,13 @@ async function layerRequirements() {
}

try {
await zipRequirements.call(this)
await createLayers.call(this)
if (this.options.layer) {
await zipRequirements.call(this)
await createLayers.call(this)
}
if (hasNamedLayers) {
await createNamedLayers.call(this)
}
} finally {
layerProgress && layerProgress.remove()
}
Expand Down
Loading
Loading