Skip to content

Commit 67d939f

Browse files
OctaveLaroseOctaveLarose
committed
No longer materializing source sections by default
1 parent a500200 commit 67d939f

12 files changed

Lines changed: 107 additions & 62 deletions

File tree

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_modulefinder.txt‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -560,11 +560,15 @@ protected CallTarget parse(ParsingRequest request) {
560560
}
561561

562562
public RootCallTarget callTargetFromBytecode(Source source, CodeUnit code) {
563+
return callTargetFromBytecode(source, code, source.isInternal());
564+
}
565+
566+
public RootCallTarget callTargetFromBytecode(Source source, CodeUnit code, boolean isInternal) {
563567
RootNode rootNode;
564568
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
565-
rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(this, source);
569+
rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(this, isInternal);
566570
} else {
567-
rootNode = PBytecodeRootNode.create(this, (BytecodeCodeUnit) code, source, source.isInternal());
571+
rootNode = PBytecodeRootNode.create(this, (BytecodeCodeUnit) code, source, isInternal);
568572
}
569573

570574
return PythonUtils.getOrCreateCallTarget(rootNode);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -655,10 +655,11 @@ private static PCode createCode(PythonContext context, FrozenInfo info) {
655655
// Fallthrough
656656
}
657657
TruffleFile originalFile = file;
658+
boolean isInternal = PythonLanguage.shouldMarkSourceInternal(context);
658659
Source source = context.getLanguage().getOrCreateSource((ignored -> {
659660
Source newSource = Source.newBuilder(PythonLanguage.ID, "", codeName) //
660661
.content(Source.CONTENT_NONE) //
661-
.internal(PythonLanguage.shouldMarkSourceInternal(context)) //
662+
.internal(isInternal) //
662663
.mimeType(PythonLanguage.MIME_TYPE).build();
663664
PythonLanguage language = context.getLanguage();
664665
if (originalFile != null) {
@@ -668,7 +669,7 @@ private static PCode createCode(PythonContext context, FrozenInfo info) {
668669
}), codeName);
669670
RootCallTarget callTarget = (RootCallTarget) context.getLanguage().cacheCode(
670671
new PythonLanguage.CodeCacheKey(info.origName, System.identityHashCode(info.code)),
671-
() -> context.getLanguage().callTargetFromBytecode(source, info.code));
672+
() -> context.getLanguage().callTargetFromBytecode(source, info.code, isInternal));
672673
/*
673674
* Setting the original filename as the co_filename is a deviance from CPython, but it's
674675
* more user friendly and lets us freeze more modules without it being too visible.

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java‎

Lines changed: 56 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -475,7 +475,7 @@ public int read(byte[] b, int off, int len) {
475475
int depth = 0;
476476
long cacheKey;
477477
TruffleFile bytecodeFile;
478-
TruffleFile sourceFile;
478+
SourceReference sourceReference;
479479
// Offset of the buffer in parent buffer in nested deserializations
480480
int baseOffset;
481481

@@ -507,23 +507,22 @@ public int read(byte[] b, int off, int len) {
507507
}
508508

509509
Marshal(PythonLanguage language, byte[] in, int length, long cacheKey) {
510-
this(language, SerializationUtils.createByteBufferDataInput(ByteBuffer.wrap(in, 0, length)), null, null, 0);
510+
this(language, SerializationUtils.createByteBufferDataInput(ByteBuffer.wrap(in, 0, length)), null, 0);
511511
this.cacheKey = cacheKey;
512512
}
513513

514514
Marshal(PythonLanguage language, byte[] in, int length, long cacheKey, TruffleFile bytecodeFile, int baseOffset) {
515-
this(language, SerializationUtils.createByteBufferDataInput(ByteBuffer.wrap(in, 0, length)), null, bytecodeFile, baseOffset);
515+
this(language, SerializationUtils.createByteBufferDataInput(ByteBuffer.wrap(in, 0, length)), bytecodeFile, baseOffset);
516516
this.cacheKey = cacheKey;
517517
}
518518

519519
Marshal(PythonLanguage language, Object in) {
520-
this(language, new DataInputStream(new FileLikeInputStream(in)), null, null, 0);
520+
this(language, new DataInputStream(new FileLikeInputStream(in)), null, 0);
521521
}
522522

523-
Marshal(PythonLanguage language, DataInput in, Source source, TruffleFile bytecodeFile, int baseOffset) {
523+
Marshal(PythonLanguage language, DataInput in, TruffleFile bytecodeFile, int baseOffset) {
524524
this.language = language;
525525
this.in = in;
526-
this.source = source;
527526
this.refList = new ArrayList<>();
528527
this.version = -1;
529528
this.outData = null;
@@ -949,7 +948,7 @@ private void writeComplexObject(Object v, int flag) {
949948
}
950949
} else if (v instanceof Source s) {
951950
writeByte(TYPE_DSL_SOURCE | flag);
952-
setSource(s);
951+
writeSource(s);
953952
} else {
954953
PythonBufferAcquireLibrary acquireLib = PythonBufferAcquireLibrary.getFactory().getUncached(v);
955954
if (acquireLib.hasBuffer(v)) {
@@ -1166,7 +1165,7 @@ private Object readObject(int type, AddRefAndReturn addRef) throws NumberFormatE
11661165
case TYPE_GRAALPYTHON_DSL_CODE_UNIT:
11671166
return addRef.run(readBytecodeDSLCodeUnit());
11681167
case TYPE_DSL_SOURCE:
1169-
return getSource();
1168+
return addRef.run(readSource());
11701169
case TYPE_DSL_EMPTY_KEYWORDS:
11711170
return PKeyword.EMPTY_KEYWORDS;
11721171
case TYPE_ARRAY:
@@ -1182,6 +1181,18 @@ private void writeString(TruffleString v) {
11821181
writeBytes(ba.getArray(), ba.getOffset(), ba.getLength());
11831182
}
11841183

1184+
private void writeSource(Source source) {
1185+
writeString(TruffleString.fromJavaStringUncached(source.getName(), TS_ENCODING));
1186+
}
1187+
1188+
private Source readSource() {
1189+
String name = readString(false).toJavaStringUncached();
1190+
if (sourceReference == null) {
1191+
sourceReference = new SourceReference(name);
1192+
}
1193+
return sourceReference.getSource(language);
1194+
}
1195+
11851196
private TruffleString readString(boolean intern) {
11861197
int sz = readInt();
11871198
if (sz == 0) {
@@ -1338,24 +1349,6 @@ private Object[] readObjectArray() {
13381349
return a;
13391350
}
13401351

1341-
private void setSource(Source s) {
1342-
if (source == null) {
1343-
source = s;
1344-
} else if (source != s) {
1345-
throw CompilerDirectives.shouldNotReachHere("attempted to serialize with multiple Source objects");
1346-
}
1347-
}
1348-
1349-
private Source getSource() {
1350-
if (source != null) {
1351-
return source;
1352-
} else {
1353-
// This should never happen when deserializing a bytecode DSL code unit, but could
1354-
// happen if the user tries to deserialize arbitrary bytes.
1355-
throw new MarshalError(ValueError, ErrorMessages.BAD_MARSHAL_DATA);
1356-
}
1357-
}
1358-
13591352
private CodeUnit readCodeUnit() {
13601353
int codeUnitType = readByte();
13611354
return switch (codeUnitType) {
@@ -1450,7 +1443,7 @@ private BytecodeDSLCodeUnit readBytecodeDSLCodeUnit() {
14501443
int yieldFromGeneratorIndex = readInt();
14511444
int instrumentationDataIndex = readInt();
14521445

1453-
BytecodeSupplier provider = new BytecodeSupplier(serialized, bytecodeFile, bytecodeOffset, bytecodeSize, cacheKey);
1446+
BytecodeSupplier provider = new BytecodeSupplier(serialized, bytecodeFile, sourceReference, bytecodeOffset, bytecodeSize, cacheKey);
14541447
return new BytecodeDSLCodeUnit(name, qualname, argCount, kwOnlyArgCount, positionalOnlyArgCount, flags, names, varnames, cellvars, freevars, cell2arg, constants,
14551448
startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, provider);
14561449
}
@@ -1545,8 +1538,8 @@ private PCode readCode() {
15451538
String jName = code.qualname.toJavaStringUncached();
15461539
Source source = Source.newBuilder(PythonLanguage.ID, "", jName).content(Source.CONTENT_NONE).build();
15471540
PythonLanguage language = this.language;
1548-
if (sourceFile != null) {
1549-
language.registerOriginalFile(source, sourceFile);
1541+
if (sourceReference != null && sourceReference.sourceFile != null) {
1542+
language.registerOriginalFile(source, sourceReference.sourceFile);
15501543
}
15511544
return language.callTargetFromBytecode(source, code);
15521545
};
@@ -1587,36 +1580,65 @@ public static CodeUnit deserializeCodeUnit(Node node, PythonLanguage language, b
15871580
}
15881581
}
15891582

1583+
private static final class SourceReference {
1584+
private String name;
1585+
private final TruffleFile sourceFile;
1586+
private Source source;
1587+
1588+
SourceReference(String name) {
1589+
this.name = name;
1590+
this.sourceFile = null;
1591+
}
1592+
1593+
SourceReference(TruffleFile sourceFile) {
1594+
this.sourceFile = sourceFile;
1595+
}
1596+
1597+
synchronized Source getSource(PythonLanguage language) {
1598+
if (source == null) {
1599+
String sourceName = sourceFile != null ? sourceFile.getPath() : name;
1600+
source = Source.newBuilder(PythonLanguage.ID, "", sourceName).content(Source.CONTENT_NONE).build();
1601+
if (sourceFile != null) {
1602+
language.registerOriginalFile(source, sourceFile);
1603+
}
1604+
}
1605+
return source;
1606+
}
1607+
}
1608+
15901609
public static class BytecodeSupplier extends BytecodeDSLCodeUnit.BytecodeSupplier {
15911610
private byte[] serialized;
15921611
// Original file for reparsing
15931612
private final TruffleFile bytecodeFile;
1613+
private final SourceReference sourceReference;
15941614
// Offset within the bytecode file, points directly at the start of serialized bytecode
15951615
private final int bytecodeOffset;
15961616
private final int bytecodeSize;
15971617
private final long cacheKey;
15981618

1599-
public BytecodeSupplier(byte[] serialized, TruffleFile bytecodeFile, int bytecodeOffset, int bytecodeSize, long cacheKey) {
1619+
public BytecodeSupplier(byte[] serialized, TruffleFile bytecodeFile, SourceReference sourceReference, int bytecodeOffset, int bytecodeSize, long cacheKey) {
16001620
this.serialized = serialized;
16011621
this.bytecodeFile = bytecodeFile;
1622+
this.sourceReference = sourceReference;
16021623
this.bytecodeOffset = bytecodeOffset;
16031624
this.bytecodeSize = bytecodeSize;
16041625
this.cacheKey = cacheKey;
16051626
}
16061627

16071628
@Override
1608-
public PBytecodeDSLRootNode createRootNode(PythonLanguage language, Source source) {
1629+
public PBytecodeDSLRootNode createRootNode(PythonLanguage language) {
16091630
BytecodeRootNodes<PBytecodeDSLRootNode> deserialized;
16101631
try {
1611-
deserialized = PBytecodeDSLRootNodeGen.deserialize(language, BytecodeConfig.WITH_SOURCE,
1632+
deserialized = PBytecodeDSLRootNodeGen.deserialize(language, BytecodeConfig.DEFAULT,
16121633
() -> SerializationUtils.createByteBufferDataInput(ByteBuffer.wrap(getBytecode())),
16131634
/*
16141635
* NB: Since a DSL node may reparse multiple times, we cannot reuse
16151636
* a common Marshal object across calls (each call may take a
16161637
* different buffer).
16171638
*/
16181639
(deserializerContext, buffer) -> {
1619-
Marshal marshal = new Marshal(language, buffer, source, bytecodeFile, bytecodeOffset);
1640+
Marshal marshal = new Marshal(language, buffer, bytecodeFile, bytecodeOffset);
1641+
marshal.sourceReference = sourceReference;
16201642
marshal.cacheKey = cacheKey;
16211643
return marshal.readObject();
16221644
});
@@ -1699,7 +1721,7 @@ public ReparseError(String message) {
16991721
@TruffleBoundary
17001722
public static Object fromBytecodeFile(PythonLanguage language, TruffleFile bytecodeFile, TruffleFile sourceFile, byte[] bytes, int offset, int length, long cacheKey) throws IOException {
17011723
MarshalModuleBuiltins.Marshal marshal = new MarshalModuleBuiltins.Marshal(language, bytes, length + offset, cacheKey, bytecodeFile, 0);
1702-
marshal.sourceFile = sourceFile;
1724+
marshal.sourceReference = sourceFile == null ? null : new SourceReference(sourceFile);
17031725
marshal.in.skipBytes(offset);
17041726
return marshal.readObject();
17051727
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeBuiltins.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,7 @@ static Object lines(PCode self) {
396396
}
397397

398398
private static List<PTuple> computeLinesForBytecodeDSLInterpreter(PBytecodeDSLRootNode root) {
399-
BytecodeNode bytecodeNode = root.getBytecodeNode();
399+
BytecodeNode bytecodeNode = root.getBytecodeNode().ensureSourceInformation();
400400
List<int[]> triples = new ArrayList<>();
401401
SourceInformationTree sourceInformationTree = bytecodeNode.getSourceInformationTree();
402402
assert sourceInformationTree.getSourceSection() != null;
@@ -495,7 +495,11 @@ Object positions(PCode self) {
495495
List<PTuple> lines = new ArrayList<>();
496496
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
497497
PBytecodeDSLRootNode rootNode = (PBytecodeDSLRootNode) self.getRootNodeForExtraction();
498-
for (Instruction instruction : rootNode.getBytecodeNode().getInstructions()) {
498+
BytecodeNode bytecodeNode = rootNode.getBytecodeNode();
499+
if (!bytecodeNode.hasSourceInformation()) {
500+
bytecodeNode = bytecodeNode.ensureSourceInformation();
501+
}
502+
for (Instruction instruction : bytecodeNode.getInstructions()) {
499503
if (instruction.isInstrumentation()) {
500504
// Skip instrumented instructions. The co_positions array should agree
501505
// with the logical instruction index.

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeNodes.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ private static RootCallTarget deserializeForBytecodeInterpreter(PythonLanguage l
181181
if (code.flags != flags) {
182182
code = code.withFlags(flags);
183183
}
184-
rootNode = code.createRootNode(language, PythonUtils.createFakeSource());
184+
rootNode = code.createRootNode(language, false);
185185
} else {
186186
BytecodeCodeUnit code = (BytecodeCodeUnit) codeUnit;
187187
if (cellvars != null && !Arrays.equals(code.cellvars, cellvars) || freevars != null && !Arrays.equals(code.freevars, freevars) || flags != code.flags) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,12 @@ private static TruffleString[] extractCellVars(RootNode rootNode) {
211211
@TruffleBoundary
212212
public static TruffleString extractFileName(RootNode rootNode) {
213213
RootNode funcRootNode = rootNodeForExtraction(rootNode);
214-
String fileName = getSourceSectionFileName(funcRootNode.getSourceSection());
214+
SourceSection sourceSection = funcRootNode.getSourceSection();
215+
if (sourceSection == null && funcRootNode instanceof PBytecodeDSLRootNode bytecodeDSLRootNode) {
216+
bytecodeDSLRootNode.getRootNodes().ensureSourceInformation();
217+
sourceSection = funcRootNode.getSourceSection();
218+
}
219+
String fileName = getSourceSectionFileName(sourceSection);
215220
if (fileName != null) {
216221
return toInternedTruffleStringUncached(fileName);
217222
}
@@ -505,7 +510,7 @@ public PCode getOrCreateChildCode(int index, BytecodeDSLCodeUnit codeUnit) {
505510
private PCode createCode(BytecodeDSLCodeUnit codeUnit) {
506511
PBytecodeDSLRootNode outerRootNode = (PBytecodeDSLRootNode) getRootNodeForExtraction();
507512
PythonLanguage language = outerRootNode.getLanguage();
508-
RootCallTarget callTarget = language.createCachedCallTarget(l -> codeUnit.createRootNode(l, outerRootNode.getSource()), codeUnit);
513+
RootCallTarget callTarget = language.createCachedCallTarget(l -> codeUnit.createRootNode(l, outerRootNode.isInternal()), codeUnit);
509514
PBytecodeDSLRootNode rootNode = (PBytecodeDSLRootNode) callTarget.getRootNode();
510515
return PFactory.createCode(language, callTarget, rootNode.getSignature(), codeUnit, getFilename());
511516
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,6 @@
157157
import com.oracle.truffle.api.bytecode.serialization.BytecodeSerializer;
158158
import com.oracle.truffle.api.debug.DebuggerTags;
159159
import com.oracle.truffle.api.instrumentation.StandardTags.StatementTag;
160-
import com.oracle.truffle.api.source.Source;
161160
import com.oracle.truffle.api.strings.TruffleString;
162161

163162
/**
@@ -413,7 +412,7 @@ private record CodeUnitKey(SSTNode node, CompilationScope scope) {
413412
private BytecodeDSLCompilerResult compileRootNode(String name, ArgumentInfo argumentInfo, SSTNode node, BytecodeParser<Builder> parser) {
414413
qualName = getNewScopeQualName(name, scopeType);
415414

416-
BytecodeRootNodes<PBytecodeDSLRootNode> nodes = PBytecodeDSLRootNodeGen.create(ctx.language, BytecodeConfig.WITH_SOURCE, parser);
415+
BytecodeRootNodes<PBytecodeDSLRootNode> nodes = PBytecodeDSLRootNodeGen.create(ctx.language, BytecodeConfig.DEFAULT, parser);
417416
List<PBytecodeDSLRootNode> nodeList = nodes.getNodes();
418417
assert nodeList.size() == 1;
419418
PBytecodeDSLRootNode rootNode = nodeList.get(0);
@@ -473,7 +472,7 @@ flags, orderedTruffleStringArray(names),
473472
new BytecodeSupplier(nodes));
474473
ctx.codeUnits.put(key, codeUnit);
475474
}
476-
rootNode.setMetadata(codeUnit, ctx.errorCallback);
475+
rootNode.setMetadata(codeUnit, ctx.errorCallback, ctx.source.isInternal());
477476
return new BytecodeDSLCompilerResult(rootNode, codeUnit);
478477
}
479478

@@ -485,7 +484,7 @@ static class BytecodeSupplier extends BytecodeDSLCodeUnit.BytecodeSupplier {
485484
}
486485

487486
@Override
488-
public PBytecodeDSLRootNode createRootNode(PythonLanguage language, Source source) {
487+
public PBytecodeDSLRootNode createRootNode(PythonLanguage language) {
489488
return nodes.getNode(0);
490489
}
491490

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@
4545
import com.oracle.truffle.api.CompilerAsserts;
4646
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
4747
import com.oracle.truffle.api.nodes.RootNode;
48-
import com.oracle.truffle.api.source.Source;
4948
import com.oracle.truffle.api.strings.TruffleString;
5049

5150
public final class BytecodeDSLCodeUnit extends CodeUnit {
@@ -67,7 +66,7 @@ public BytecodeDSLCodeUnit(TruffleString name, TruffleString qualname, int argCo
6766
}
6867

6968
public abstract static class BytecodeSupplier {
70-
public abstract PBytecodeDSLRootNode createRootNode(PythonLanguage language, Source source);
69+
public abstract PBytecodeDSLRootNode createRootNode(PythonLanguage language);
7170

7271
public abstract byte[] createSerializedBytecode(PythonLanguage language);
7372
}
@@ -79,12 +78,12 @@ public BytecodeDSLCodeUnit withFlags(int flags) {
7978
}
8079

8180
@TruffleBoundary
82-
public PBytecodeDSLRootNode createRootNode(PythonLanguage language, Source source) {
81+
public PBytecodeDSLRootNode createRootNode(PythonLanguage language, boolean isInternal) {
8382
// We must not cache deserialized root, because the code unit may be shared by multiple
8483
// engines. The caller is responsible for ensuring the caching of the resulting root node if
8584
// necessary
86-
PBytecodeDSLRootNode rootNode = supplier.createRootNode(language, source);
87-
rootNode.setMetadata(this, null);
85+
PBytecodeDSLRootNode rootNode = supplier.createRootNode(language);
86+
rootNode.setMetadata(this, null, isInternal);
8887
return rootNode;
8988
}
9089

0 commit comments

Comments
 (0)