diff --git a/biojava-integrationtest/src/test/java/org/biojava/nbio/structure/test/io/density/DensityMapIntegrationTest.java b/biojava-integrationtest/src/test/java/org/biojava/nbio/structure/test/io/density/DensityMapIntegrationTest.java new file mode 100644 index 0000000000..97bf76a294 --- /dev/null +++ b/biojava-integrationtest/src/test/java/org/biojava/nbio/structure/test/io/density/DensityMapIntegrationTest.java @@ -0,0 +1,215 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.test.io.density; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.List; + +import org.biojava.nbio.core.util.FileDownloadUtils; +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.io.density.Ccp4Header; +import org.biojava.nbio.structure.io.density.DensityFileFormat; +import org.biojava.nbio.structure.io.density.DensityMapCache; +import org.biojava.nbio.structure.io.density.DensityMapKind; +import org.biojava.nbio.structure.io.density.DensityMapRequest; +import org.biojava.nbio.structure.io.density.DensityMapResult; +import org.biojava.nbio.structure.io.density.DensityMapSource; +import org.biojava.nbio.structure.io.density.NoDensityMapException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Density fetching against the real services. + *
+ * Deliberately frugal: the entries chosen keep a full run to a couple of
+ * megabytes plus a few small metadata calls. In particular the cryo-EM path is
+ * exercised with the size limit set so low that the 116 MB map is declined
+ * before any of its body is transferred, which tests the whole resolution and
+ * guard sequence without the download.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class DensityMapIntegrationTest {
+
+ private File cacheRoot;
+ private DensityMapCache cache;
+
+ @BeforeEach
+ public void setUp() throws IOException {
+ cacheRoot = Files.createTempDirectory("bj-density-it").toFile();
+ cache = new DensityMapCache(cacheRoot.getAbsolutePath());
+ }
+
+ @AfterEach
+ public void tearDown() throws IOException {
+ FileDownloadUtils.deleteDirectory(cacheRoot.toPath());
+ }
+
+ /** The default path for an X-ray entry: the smallest source answers first. */
+ @Test
+ public void fetchesAnXrayMapFromTheFirstSourceTried() throws IOException {
+ DensityMapResult result = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
+
+ assertEquals(DensityMapSource.RCSB_VOLUME_SERVER, result.getSource());
+ assertEquals(DensityMapKind.TWO_FO_FC, result.getKind());
+ assertTrue(result.isRenderable());
+ assertFalse(result.isFromCache());
+ assertTrue(result.getFileSizeBytes() > 1024);
+ assertTrue(DensityMapResult.metaFileFor(result.getFile()).isFile(),
+ "a .meta sidecar makes the result reconstructible offline");
+
+ // second call must come from the cache without another download
+ DensityMapResult again = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
+ assertTrue(again.isFromCache());
+ assertEquals(result.getFile(), again.getFile());
+ }
+
+ /**
+ * Both map kinds come out of one download, and the difference map is presented
+ * under the companion name that makes Jmol read the other data block.
+ */
+ @Test
+ public void bothKindsShareASingleDownload() throws IOException {
+ DensityMapResult twoFoFc = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
+ DensityMapResult foFc = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.FO_FC);
+
+ assertEquals(DensityMapKind.FO_FC, foFc.getKind());
+ assertFalse(twoFoFc.getFile().equals(foFc.getFile()), "the difference map needs its own file name");
+ assertTrue(foFc.getFile().getName().contains("&diff=1"),
+ "the marker has to be in the name for Jmol to select the FO-FC block");
+ assertEquals(twoFoFc.getFileSizeBytes(), foFc.getFileSizeBytes(),
+ "both names must address the same bytes");
+ }
+
+ /** PDBe serves real CCP4 files, which the header check should recognise. */
+ @Test
+ public void pdbeServesAGenuineCcp4Map() throws IOException {
+ cache.setSourceChain(DensityMapKind.TWO_FO_FC, Arrays.asList(DensityMapSource.PDBE_CCP4));
+ DensityMapResult result = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
+
+ assertEquals(DensityMapSource.PDBE_CCP4, result.getSource());
+ assertEquals(DensityFileFormat.CCP4, result.getFormat());
+ assertTrue(Ccp4Header.isCcp4(result.getFile()), "the CCP4 stamp should be present at byte 208");
+ assertTrue(FileDownloadUtils.validateFile(result.getFile()));
+ }
+
+ /**
+ * The whole cryo-EM route: resolve the EMDB entry, pick up the author contour
+ * level, and decline the full map on size without transferring it.
+ */
+ @Test
+ public void resolvesCryoEmEntriesAndHonoursTheSizeLimit() throws IOException {
+ List
+ * The divided archive paths on files.wwpdb.org and files.rcsb.org return the content
+ * MD5 as the ETag. The flat /validation/download/ endpoint this provider now uses
+ * returns neither an ETag nor a Content-Length, and neither does the beta archive on
+ * those two hosts, so no digest can be recorded there. The size sidecar is written
+ * from the bytes actually read, so it exists either way.
+ *
+ * The digest is therefore asserted when the server offered one and skipped when it
+ * did not, rather than being required: requiring it would fail against the endpoint
+ * we use, and hard-coding the divided path would only work until the archive
+ * transition in July 2027.
+ */
+ @Test
+ public void mapCoefficientsArriveIntactAndVerifiable() throws IOException {
+ cache.setSourceEnabled(DensityMapSource.WWPDB_MAP_COEFFICIENTS, true);
+ cache.setSourceChain(DensityMapKind.TWO_FO_FC, Arrays.asList(DensityMapSource.WWPDB_MAP_COEFFICIENTS));
+
+ DensityMapResult result = cache.getDensityMap(DensityMapRequest.builder(new PdbId("1cbs"))
+ .kind(DensityMapKind.TWO_FO_FC)
+ .allowNonRenderableFormats(true)
+ .build());
+
+ assertEquals(DensityMapSource.WWPDB_MAP_COEFFICIENTS, result.getSource());
+ assertFalse(result.isRenderable(),
+ "structure factors are not a map and must not claim to be renderable");
+
+ // written from the observed byte count, so it is present whether or not the
+ // server declared a length
+ assertTrue(FileDownloadUtils.validateFile(result.getFile()),
+ "a freshly downloaded file must validate against its own sidecars");
+
+ File hashFile = new File(result.getFile().getParentFile(), result.getFile().getName() + ".hash_MD5");
+ if (hashFile.isFile()) {
+ String recorded = new String(Files.readAllBytes(hashFile.toPath()), StandardCharsets.UTF_8).trim();
+ assertTrue(FileDownloadUtils.verifyHash(result.getFile(), FileDownloadUtils.Hash.MD5, recorded),
+ "the recorded MD5 must match the file it describes");
+ } else {
+ System.out.println("No MD5 recorded for " + result.getSourceUrl()
+ + " - the server offered no usable ETag. Size validation still applies.");
+ }
+
+ // corrupt it and confirm validation actually catches it
+ Files.write(result.getFile().toPath(), new byte[] {0, 1, 2, 3});
+ assertFalse(FileDownloadUtils.validateFile(result.getFile()),
+ "a truncated file must not validate");
+ }
+}
diff --git a/biojava-structure-gui/src/main/java/demo/DemoShowElectronDensity.java b/biojava-structure-gui/src/main/java/demo/DemoShowElectronDensity.java
new file mode 100644
index 0000000000..3948605744
--- /dev/null
+++ b/biojava-structure-gui/src/main/java/demo/DemoShowElectronDensity.java
@@ -0,0 +1,80 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package demo;
+
+import org.biojava.nbio.structure.PdbId;
+import org.biojava.nbio.structure.Structure;
+import org.biojava.nbio.structure.StructureIO;
+import org.biojava.nbio.structure.align.gui.jmol.StructureAlignmentJmol;
+import org.biojava.nbio.structure.io.density.DensityMapCache;
+import org.biojava.nbio.structure.io.density.DensityMapKind;
+import org.biojava.nbio.structure.io.density.DensityMapResult;
+
+/**
+ * Shows 1CBS with its electron density drawn around the bound retinoic acid.
+ *
+ * Both maps are displayed: the 2mFo-DFc map in blue at 1 sigma, which should hug
+ * the ligand closely, and the mFo-DFc difference map as a red and green pair at
+ * 3 sigma, which for a well refined structure should show very little.
+ *
+ * Once the window is up, the map can be manipulated from the Rasmol command box
+ * at the bottom, for instance
+ *
+ * The download happens on a {@link SwingWorker} rather than the event dispatch
+ * thread. Even the smallest source runs to a few hundred kilobytes and a
+ * full-resolution map can be far larger, so fetching inline would freeze the
+ * interface for as long as it took.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class MyShowDensityListener implements ActionListener {
+
+ private static final Logger logger = LoggerFactory.getLogger(MyShowDensityListener.class);
+
+ private static final String OPTION_2FOFC = "2Fo-Fc (electron density)";
+ private static final String OPTION_FOFC = "Fo-Fc (difference)";
+ private static final String OPTION_BOTH = "Both";
+ private static final String OPTION_AUTO = "Whatever is available";
+
+ private final AbstractAlignmentJmol parent;
+
+ /**
+ * @param parent the viewer to draw into
+ */
+ public MyShowDensityListener(AbstractAlignmentJmol parent) {
+ this.parent = parent;
+ }
+
+ @Override
+ public void actionPerformed(ActionEvent e) {
+ Structure structure = parent == null ? null : parent.getStructure();
+ if (structure == null) {
+ JOptionPane.showMessageDialog(parent == null ? null : parent.getFrame(),
+ "There is no structure on screen to fetch a density map for.",
+ "Show Electron Density", JOptionPane.INFORMATION_MESSAGE);
+ return;
+ }
+
+ PdbId pdbId = structure.getPdbId();
+ if (pdbId == null) {
+ String typed = JOptionPane.showInputDialog(parent.getFrame(),
+ "This structure has no PDB ID. Enter one to look up its density map:",
+ "Show Electron Density", JOptionPane.QUESTION_MESSAGE);
+ if (typed == null || typed.trim().isEmpty()) {
+ return;
+ }
+ try {
+ pdbId = new PdbId(typed.trim());
+ } catch (IllegalArgumentException ex) {
+ JOptionPane.showMessageDialog(parent.getFrame(), typed + " is not a valid PDB ID.",
+ "Show Electron Density", JOptionPane.ERROR_MESSAGE);
+ return;
+ }
+ }
+
+ Object[] options = {OPTION_2FOFC, OPTION_FOFC, OPTION_BOTH, OPTION_AUTO};
+ Object choice = JOptionPane.showInputDialog(parent.getFrame(),
+ "Which map would you like to see for " + pdbId.getId() + "?",
+ "Show Electron Density", JOptionPane.QUESTION_MESSAGE, null, options, OPTION_2FOFC);
+ if (choice == null) {
+ return;
+ }
+
+ List
+ * Contouring follows the convention for each kind of map: the 2mFo-DFc map at
+ * 1 sigma in blue, the mFo-DFc difference map as a signed red/green pair at 3
+ * sigma, and a cryo-EM map at the level its depositors recommend when the entry
+ * states one, falling back to 3 sigma when it does not.
+ *
+ * Clipping matters for more than tidiness: contouring a whole cryo-EM grid at
+ * mesh resolution can take long enough to make the interface appear frozen.
+ *
+ * @param map the map to display
+ * @param atomSelection a Jmol atom expression such as
+ * The option order is not a matter of taste:
+ * Going through {@link File#toURI()} percent-encodes spaces and removes
+ * backslashes, which Jmol would otherwise read as escape characters in a script
+ * string. On Windows the result is a single-slash
+ * The three entries chosen exercise the three outcomes the fallback chain has to
+ * handle:
+ *
+ * This is reported through the logger and on both standard output and standard
+ * error. A user who asked for a map and silently got a coarser one from another
+ * source deserves to be told why, and a library log configuration that discards
+ * warnings should not be able to hide it.
+ *
+ * @param url the resource that was skipped
+ * @param size its size in bytes
+ * @param maxBytes the limit in bytes
+ * @param request the request being served
+ */
+ protected void reportTooLarge(URL url, long size, long maxBytes, DensityMapRequest request) {
+ String entry = request.getPdbId() != null ? request.getPdbId().getId()
+ : (request.getEmdbId() != null ? request.getEmdbId() : "?");
+ String message = String.format(
+ "BioJava density: skipping %s for %s (%,d bytes exceeds the %,d byte limit). "
+ + "Trying a smaller representation from another source; "
+ + "raise DensityMapCache.setMaxDownloadBytes() to allow it.",
+ url, entry, size, maxBytes);
+ logger.warn(message);
+ System.out.println(message);
+ System.err.println(message);
+ }
+
+ /**
+ * The
+ * This is a cheap but effective guard against a cached file that is not
+ * actually a map. A misbehaving or overloaded server can answer with an HTML
+ * error page and an HTTP 200, in which case nothing about the status code or the
+ * content length reveals the problem — but the missing
+ * PDB-keyed maps follow the divided layout the rest of BioJava already uses, so
+ * that a cache with many entries does not end up with one enormous directory:
+ *
+ * EMDB maps are keyed by EMDB identifier instead, mirroring the EMDB archive:
+ *
+ * The two-character directory needs no attention when the archive moves to
+ * extended identifiers in July 2027. It is a device for spreading files over
+ * directories, not a copy of the archive's own layout, and
+ * {@link LocalPDBDirectory#getMiddleHash(String)} counts from the right hand end
+ * of the identifier, so
+ * Deliberately not the archive's per-entry layout. A cache is not a mirror
+ * and cannot become one: the archive publishes structure factors and map
+ * coefficients but never grids, so density has to be fetched whatever else is
+ * mirrored, and writing it into a directory whose contents are an exact copy of
+ * upstream puts it at the mercy of the next
+ * The separate kind token exists for sources that deliver more than one kind of
+ * map in a single file. A density server response, for instance, carries both
+ * the 2Fo-Fc and the Fo-Fc blocks, so it is cached once under
+ * {@link #BOTH_KINDS_TOKEN} rather than downloaded and stored twice.
+ *
+ * @param cacheRoot the BioJava cache directory
+ * @param pdbId the entry
+ * @param kindToken the token naming what the file holds
+ * @param source the service it came from
+ * @param format the file format
+ * @param qualifier an extra discriminator such as a detail level, or
+ * A density server response holds both a
+ * This was verified against Jmol 14.31.10 and is unchanged in current Jmol: the
+ * relevant line in
+ * Every entry in the archive today has a four-character form, and the density
+ * services accept only that spelling. Extended-only identifiers will appear
+ * eventually; rather than refusing them, the extended spelling is passed
+ * through so that the services can start accepting it without a change here.
+ *
+ * @param pdbId the identifier
+ * @return the short spelling if available, otherwise the full one
+ */
+ public static String shortIdOrFull(PdbId pdbId) {
+ return pdbId.getId(true);
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityFileFormat.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityFileFormat.java
new file mode 100644
index 0000000000..11cbb0e27e
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityFileFormat.java
@@ -0,0 +1,89 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+/**
+ * The file format a density map was delivered in.
+ *
+ * The important distinction here is {@link #isJmolLoadable()}. Most of these
+ * formats are sampled grids that a viewer can contour directly; map
+ * coefficients are not, they are structure factors that require a Fourier
+ * transform first. Handing the latter to a viewer produces nothing at all, so
+ * the difference has to be visible to callers.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public enum DensityFileFormat {
+
+ /** A CCP4/MRC sampled map, as served pre-computed by PDBe. */
+ CCP4(".ccp4", true),
+
+ /** A gzipped CCP4/MRC map, the form EMDB distributes its primary maps in. */
+ CCP4_GZ(".map.gz", true),
+
+ /** A BinaryCIF volume slice from a Mol* density server. */
+ BCIF_VOLUME(".bcif", true),
+
+ /** A text CIF volume slice from a Mol* density server. */
+ CIF_VOLUME(".cif", true),
+
+ /**
+ * Structure-factor amplitudes and phases in mmCIF, as published with the wwPDB
+ * validation reports. Not a density grid. A Fourier transform (for
+ * example
+ * Cached files live under the BioJava cache directory (
+ * Source order. Sources are tried smallest-adequate-first, because they
+ * differ enormously in size for the same entry and the smallest is usually
+ * perfectly adequate to look at. For 1cbs a density-server slice is roughly a
+ * tenth the size of the equivalent pair of CCP4 files; for the cryo-EM entry
+ * behind EMD-0262 it is a few hundred kilobytes against 106 MB. Anything
+ * that cannot be displayed at all is tried last, and
+ * {@link DensityMapSource#WWPDB_MAP_COEFFICIENTS} is disabled altogether by
+ * default for that reason. Override with {@link #setSourceChain(DensityMapKind,
+ * java.util.List)} or {@link #setSourceEnabled(DensityMapSource, boolean)}.
+ *
+ * Failures. A source that has nothing for an entry is skipped and the next
+ * is tried; if every source is exhausted, {@link NoDensityMapException} is thrown
+ * carrying the reason from each one, so a caller can explain what happened.
+ * Genuine transport failures abort the chain instead, so that a network outage is
+ * never reported as "this entry has no density".
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class DensityMapCache {
+
+ private static final Logger logger = LoggerFactory.getLogger(DensityMapCache.class);
+
+ /**
+ * Default ceiling on a single download, 256 MiB. Exceeding it is not an
+ * error: the chain moves on to a source that offers a smaller representation.
+ * Set to 0 to remove the limit.
+ */
+ public static final long DEFAULT_MAX_DOWNLOAD_BYTES = 256L * 1024 * 1024;
+
+ /** Order in which sources are tried for X-ray and neutron entries. */
+ public static final List
+ * {@link DensityMapSource#WWPDB_MAP_COEFFICIENTS} is disabled by default
+ * because it delivers structure factors rather than a map; enable it explicitly
+ * if you want the archival form and are prepared to run a Fourier transform.
+ *
+ * @param source the source
+ * @param enabled whether to try it
+ */
+ public void setSourceEnabled(DensityMapSource source, boolean enabled) {
+ if (enabled) {
+ disabled.remove(source);
+ } else {
+ disabled.add(source);
+ }
+ }
+
+ /**
+ * Replaces the provider used for a source. Mainly a testing seam, but also the
+ * way to plug in a site-local mirror.
+ *
+ * @param provider the provider to use
+ */
+ public void registerProvider(DensityMapProvider provider) {
+ register(provider);
+ applySettingsToProviders();
+ }
+
+ /** @return the resolver used to map PDB entries to EMDB entries */
+ public EmdbEntryResolver getEmdbResolver() {
+ return emdbResolver;
+ }
+
+ /** @param resolver the resolver used to map PDB entries to EMDB entries */
+ public void setEmdbResolver(EmdbEntryResolver resolver) {
+ this.emdbResolver = resolver;
+ applySettingsToProviders();
+ }
+
+ /**
+ * Fetches a density map, using the cache where possible.
+ *
+ * @param pdbId the entry
+ * @param kind the kind of map, or {@link DensityMapKind#AUTO} to take whatever
+ * the entry has
+ * @return the map
+ * @throws NoDensityMapException if no enabled source has a map for this entry
+ * @throws IOException on transport failure
+ */
+ public DensityMapResult getDensityMap(PdbId pdbId, DensityMapKind kind) throws IOException {
+ return getDensityMap(DensityMapRequest.builder(pdbId).kind(kind).build());
+ }
+
+ /**
+ * Fetches a density map for a PDB or EMDB identifier.
+ *
+ * @param id a PDB identifier, or an EMDB identifier such as
+ * Knowing the structure lets the experimental method be read directly rather
+ * than guessed, so a cryo-EM entry goes straight to the EM sources instead of
+ * trying the X-ray ones first. No extra network request is involved.
+ *
+ * @param structure the structure
+ * @param kind the kind of map wanted
+ * @return the map
+ * @throws NoDensityMapException if no enabled source has a map for this entry
+ * @throws IOException on transport failure
+ */
+ public DensityMapResult getDensityMap(Structure structure, DensityMapKind kind) throws IOException {
+ PdbId pdbId = structure.getPdbId();
+ if (pdbId == null) {
+ throw new IOException("The structure has no PDB ID, so no density map can be looked up for it.");
+ }
+ DensityMapRequest request = DensityMapRequest.builder(pdbId).kind(kind).build();
+ return getDensityMap(request, kindOrderFor(structure, kind));
+ }
+
+ /**
+ * Fetches a density map.
+ *
+ * @param request what is wanted
+ * @return the map
+ * @throws NoDensityMapException if no enabled source has a map for this entry
+ * @throws IOException on transport failure
+ */
+ public DensityMapResult getDensityMap(DensityMapRequest request) throws IOException {
+ return getDensityMap(request, request.getKind().resolve());
+ }
+
+ /**
+ * As {@link #getDensityMap(PdbId, DensityMapKind)} but returning an empty
+ * {@link Optional} rather than throwing when nothing is available. Genuine
+ * transport failures are still logged and swallowed, so use this only where
+ * "no map" and "could not reach the server" need not be told apart.
+ *
+ * @param pdbId the entry
+ * @param kind the kind of map wanted
+ * @return the map, if one could be obtained
+ */
+ public Optional
+ * A density server returns voxels and nothing else, but an EM map is
+ * conventionally displayed at the level its depositors chose rather than at a
+ * multiple of sigma, so a viewer needs that number whichever source the map came
+ * from. It costs one small metadata request, cached thereafter.
+ */
+ private DensityMapResult withContourLevel(DensityMapResult result) {
+ if (result.getKind() != DensityMapKind.EM || result.getRecommendedContourLevel() != null
+ || result.getEmdbId() == null || emdbResolver == null) {
+ return result;
+ }
+ EmdbEntryInfo info = emdbResolver.getEntryInfo(result.getEmdbId());
+ if (info == null || (info.getRecommendedContourLevel() == null && info.getSigma() == null)) {
+ return result;
+ }
+ DensityMapResult enriched = new DensityMapResult(result.getFile(), result.getSource(), result.getFormat(),
+ result.getKind(), result.getPdbId(), result.getEmdbId(), result.getSourceUrl(), result.isFromCache(),
+ info.getRecommendedContourLevel(), info.getSigma());
+ enriched.writeMeta();
+ return enriched;
+ }
+
+ private String resolveEmdbId(DensityMapRequest request) {
+ if (request.getPdbId() == null || emdbResolver == null) {
+ return null;
+ }
+ List
+ * Implementations are combined into an ordered chain by {@link DensityMapCache},
+ * which tries each in turn until one produces a map. The contract around
+ * exceptions is what makes that chain safe:
+ *
+ * Instances are immutable; build them with {@link #builder(PdbId)} or
+ * {@link #builder(String)}.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class DensityMapRequest {
+
+ private final PdbId pdbId;
+ private final String emdbId;
+ private final DensityMapKind kind;
+ private final FetchBehavior fetchBehavior;
+ private final File cacheDir;
+ private final boolean allowNonRenderableFormats;
+ private final long maxDownloadBytes;
+ private final List
+ * Which source answered matters to the caller and is therefore part of the
+ * result: the file might be a full CCP4 grid, a downsampled volume slice, or
+ * — if non-renderable formats were allowed — a set of structure
+ * factors that must be Fourier-transformed before anything can be drawn. See
+ * {@link #isRenderable()}.
+ *
+ * Every field is persisted to a
+ * These differ enormously in size for the same entry, which is why more than one
+ * is supported. For PDB entry 1cbs a density-server slice at the coarsest detail
+ * level is about 210 kB and contains both the 2Fo-Fc and Fo-Fc maps, where
+ * the two pre-computed CCP4 files come to about 2.1 MB together. For cryo-EM
+ * the gap is far wider: the primary map of EMD-0262 is about 106 MB, against
+ * roughly 480 kB for the equivalent density-server slice.
+ *
+ * Note that RCSB's own
+ * Cryo-EM primary maps in particular can be very large — hundreds of
+ * megabytes is common and gigabyte maps exist — while a downsampled slice
+ * of the same map from a density server is usually a fraction of a percent of
+ * that size and quite adequate for display. Rather than failing, the fallback
+ * chain treats this as "try the next source", which is exactly why the density
+ * servers are tried ahead of the full-resolution archives.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class DensityMapTooLargeException extends IOException {
+
+ private static final long serialVersionUID = 1L;
+
+ private final long sizeBytes;
+ private final long limitBytes;
+
+ /**
+ * @param url the resource that was too large
+ * @param sizeBytes its size, or a negative value if only a lower bound is known
+ * @param limitBytes the configured limit
+ */
+ public DensityMapTooLargeException(String url, long sizeBytes, long limitBytes) {
+ super(String.format("%s is %s, which exceeds the %s download limit.",
+ url, describe(sizeBytes), describe(limitBytes)));
+ this.sizeBytes = sizeBytes;
+ this.limitBytes = limitBytes;
+ }
+
+ private static String describe(long bytes) {
+ if (bytes < 0) {
+ return "of unknown size";
+ }
+ if (bytes < 1024) {
+ return bytes + " B";
+ }
+ double value = bytes;
+ String[] units = {"kB", "MB", "GB", "TB"};
+ int unit = -1;
+ while (value >= 1024 && unit < units.length - 1) {
+ value /= 1024;
+ unit++;
+ }
+ return String.format("%.1f %s", value, units[unit]);
+ }
+
+ /**
+ * @return the size of the resource in bytes, or a negative value if unknown
+ */
+ public long getSizeBytes() {
+ return sizeBytes;
+ }
+
+ /**
+ * @return the configured limit in bytes
+ */
+ public long getLimitBytes() {
+ return limitBytes;
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryInfo.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryInfo.java
new file mode 100644
index 0000000000..7c8b0dd302
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryInfo.java
@@ -0,0 +1,85 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+/**
+ * The few facts about an EMDB entry's primary map that matter when fetching and
+ * displaying it.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class EmdbEntryInfo {
+
+ private final String emdbId;
+ private final Double recommendedContourLevel;
+ private final Double sigma;
+ private final Long mapSizeBytes;
+
+ /**
+ * @param emdbId the entry identifier, canonical form
+ * @param recommendedContourLevel the author-recommended contour level in absolute
+ * map units, or
+ * The primary lookup is EMDB's own search API, which returns the identifier and
+ * the author-recommended contour level together in a single small CSV response.
+ * That is also the route Jmol uses, so BioJava's answer matches what a user sees
+ * elsewhere. If it fails, RCSB's entry API is consulted for the identifier alone.
+ *
+ * The experimental method is deliberately not inferred from a structure's
+ * resolution: BioJava is known to mis-parse resolution for some cryo-EM entries
+ * (biojava/biojava#1000), so the presence of an EMDB identifier is the reliable
+ * signal.
+ *
+ * Answers are cached on disk. Under
+ * {@link FetchBehavior#LOCAL_ONLY} the cached answer is used whatever its age and
+ * no connection is opened.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class EmdbEntryResolver {
+
+ private static final Logger logger = LoggerFactory.getLogger(EmdbEntryResolver.class);
+
+ /**
+ * Default template for the EMDB search that maps a PDB entry to the EM
+ * reconstructions it was fitted into, returning the contour level as well.
+ */
+ public static final String DEFAULT_SEARCH_URL_TEMPLATE =
+ "https://www.ebi.ac.uk/emdb/api/search/fitted_pdbs:{pdbid_lc}?fl=emdb_id,map_contour_level_value&wt=csv";
+
+ /** Default template for the EMDB entry map metadata. */
+ public static final String DEFAULT_MAP_INFO_URL_TEMPLATE =
+ "https://www.ebi.ac.uk/emdb/api/entry/map/{emdb_id}";
+
+ /** Default template for the RCSB entry record, used as a fallback. */
+ public static final String DEFAULT_RCSB_ENTRY_URL_TEMPLATE =
+ "https://data.rcsb.org/rest/v1/core/entry/{pdbid_lc}";
+
+ private static final int TIMEOUT_MILLIS = 30000;
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private static String searchUrlTemplate = DEFAULT_SEARCH_URL_TEMPLATE;
+ private static String mapInfoUrlTemplate = DEFAULT_MAP_INFO_URL_TEMPLATE;
+ private static String rcsbEntryUrlTemplate = DEFAULT_RCSB_ENTRY_URL_TEMPLATE;
+
+ private File cacheRoot;
+ private FetchBehavior fetchBehavior = FetchBehavior.FETCH_FILES;
+ private int mappingMaxAgeDays = 30;
+
+ /**
+ * @param cacheRoot the BioJava cache directory
+ */
+ public EmdbEntryResolver(File cacheRoot) {
+ this.cacheRoot = cacheRoot;
+ }
+
+ /** @param cacheRoot the BioJava cache directory */
+ public void setCacheRoot(File cacheRoot) {
+ this.cacheRoot = cacheRoot;
+ }
+
+ /** @param fetchBehavior how aggressively to refresh cached answers */
+ public void setFetchBehavior(FetchBehavior fetchBehavior) {
+ this.fetchBehavior = fetchBehavior == null ? FetchBehavior.FETCH_FILES : fetchBehavior;
+ }
+
+ /**
+ * @param days how long a cached mapping stays fresh. Mappings change only when
+ * an entry is re-released, so this can be generous.
+ */
+ public void setMappingMaxAgeDays(int days) {
+ this.mappingMaxAgeDays = days;
+ }
+
+ /** @param template the EMDB search URL template */
+ public static void setSearchUrlTemplate(String template) {
+ searchUrlTemplate = template == null ? DEFAULT_SEARCH_URL_TEMPLATE : template;
+ }
+
+ /** @param template the EMDB map metadata URL template */
+ public static void setMapInfoUrlTemplate(String template) {
+ mapInfoUrlTemplate = template == null ? DEFAULT_MAP_INFO_URL_TEMPLATE : template;
+ }
+
+ /** @param template the RCSB entry URL template used as a fallback */
+ public static void setRcsbEntryUrlTemplate(String template) {
+ rcsbEntryUrlTemplate = template == null ? DEFAULT_RCSB_ENTRY_URL_TEMPLATE : template;
+ }
+
+ /** Restores all default URL templates. */
+ public static void resetToDefaults() {
+ searchUrlTemplate = DEFAULT_SEARCH_URL_TEMPLATE;
+ mapInfoUrlTemplate = DEFAULT_MAP_INFO_URL_TEMPLATE;
+ rcsbEntryUrlTemplate = DEFAULT_RCSB_ENTRY_URL_TEMPLATE;
+ }
+
+ /**
+ * The EMDB entries a PDB entry was fitted into.
+ *
+ * @param pdbId the PDB entry
+ * @return the EMDB identifiers, most relevant first; empty if the entry is not
+ * an EM structure or has no associated map
+ */
+ public List
+ * The response is cached verbatim, so asking twice costs one request.
+ *
+ * @param emdbId the EMDB entry, in any accepted form
+ * @return the metadata, or
+ * These are gzipped CCP4/MRC files, and they are big: the primary map of
+ * EMD-0262 is about 106 MB, and gigabyte maps exist. For most display
+ * purposes a slice from a density server is a far better trade — a few
+ * hundred kilobytes for the same entry — which is why
+ * {@link DensityMapCache} tries {@link VolumeServerProvider} first and only falls
+ * back here. Use this source when the full sampling genuinely matters.
+ *
+ * The size limit is checked against the size EMDB itself reports before any of
+ * the body is transferred, so exceeding it costs one small metadata request
+ * rather than a partial download.
+ *
+ * Jmol recognises gzip from the file's magic bytes, so the cached
+ *
+ * This is a routine outcome rather than an error: not every entry has density.
+ * Structures deposited without structure factors have none at all (4HHB, for
+ * instance), and a cryo-EM entry has no X-ray maps by construction. The
+ * per-source reasons are kept so that a caller — particularly a user
+ * interface — can explain why rather than just reporting a failure.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class NoDensityMapException extends IOException {
+
+ private static final long serialVersionUID = 1L;
+
+ private final PdbId pdbId;
+ private final DensityMapKind kind;
+ private final Map
+ * These are full-resolution sampled grids, one file per map kind, and are the
+ * source Jmol itself uses for its built-in map shortcuts. They are larger than a
+ * density-server slice — about 1 MB per map for a small entry such as
+ * 1cbs — but need no interpretation beyond contouring.
+ *
+ * The service accepts only the lower-case four-character spelling of an
+ * identifier;
+ * The recognised placeholders are:
+ *
+ *
+ * This is deliberately separate from the template mechanism in
+ * {@code DownloadChemCompProvider}, which resolves a single chemical-component
+ * identifier with optional substring indices. The two solve different problems:
+ * here several distinct values are substituted by name.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class UrlTemplates {
+
+ private static final Pattern PLACEHOLDER = Pattern.compile("\\{([a-zA-Z_]+)\\}");
+
+ private UrlTemplates() {
+ }
+
+ /**
+ * Expands a template against a set of named values.
+ *
+ * @param template the template string
+ * @param values the values, keyed by placeholder name without the braces
+ * @return the expanded string
+ */
+ public static String expand(String template, Map
+ * This is usually the most economical source by a wide margin. For PDB entry
+ * 1cbs the coarsest slice is about 210 kB against roughly 2.1 MB for
+ * the two pre-computed CCP4 files, and for cryo-EM the difference is far larger
+ * still: about 480 kB against the 106 MB primary map of EMD-0262. A
+ * single response carries both the 2Fo-Fc and Fo-Fc data blocks for X-ray
+ * entries.
+ *
+ * Caveats worth knowing. Responses are chunked, with no
+ *
+ * The companion is a hard link where the filesystem allows one, so the second
+ * name costs no additional space; a copy is only made if linking is refused.
+ */
+ private DensityMapResult presentAsDifferenceMap(DensityMapResult result) throws IOException {
+ File marker = DensityCacheLayout.differenceMarkerFile(result.getFile());
+ if (!marker.isFile() || marker.length() != result.getFile().length()) {
+ Files.deleteIfExists(marker.toPath());
+ try {
+ Files.createLink(marker.toPath(), result.getFile().toPath());
+ } catch (IOException | UnsupportedOperationException e) {
+ Files.copy(result.getFile().toPath(), marker.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+ return new DensityMapResult(marker, result.getSource(), result.getFormat(), result.getKind(),
+ result.getPdbId(), result.getEmdbId(), result.getSourceUrl(), result.isFromCache(),
+ result.getRecommendedContourLevel(), result.getSigma());
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/WwpdbMapCoefficientsProvider.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/WwpdbMapCoefficientsProvider.java
new file mode 100644
index 0000000000..bd1b066a64
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/WwpdbMapCoefficientsProvider.java
@@ -0,0 +1,219 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
+import org.biojava.nbio.structure.PdbId;
+
+/**
+ * Fetches the map coefficients published alongside the wwPDB validation reports.
+ *
+ * These are not density maps. They are structure-factor amplitudes and
+ * phases in mmCIF, exactly as used to produce the pictures in a validation
+ * report, and a Fourier transform is required before anything can be drawn from
+ * them —
+ * They are supported because this is the route RCSB documents since
+ *
+ * One useful property: these servers return the content MD5 as the HTTP
+ *
+ * The URLs are built against the documented download endpoint, which resolves an
+ * entry by file name, rather than against the divided archive path. This is the
+ * only provider here that ever had a choice — the density servers and the
+ * EMDB archive are addressed by identifier already — and it matters because
+ * the PDB moves to extended identifiers and a per-entry directory layout in July
+ * 2027. A name survives that move; a constructed directory path does not. Mirrors
+ * that publish directories instead of an endpoint are still reachable, through
+ * {@link #DIVIDED_TWO_FO_FC_TEMPLATE} and {@link #ENTRIES_TWO_FO_FC_TEMPLATE}.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class WwpdbMapCoefficientsProvider extends AbstractDensityMapProvider {
+
+ /** Default base URL, the wwPDB validation report download endpoint. */
+ public static final String DEFAULT_SERVER_URL = "https://files.wwpdb.org/validation/download/";
+
+ /** An RCSB mirror serving byte-identical files. */
+ public static final String RCSB_MIRROR_URL = "https://files.rcsb.org/validation/download/";
+
+ /**
+ * The wwPDB beta archive, which already holds the re-organised content that
+ * replaces the current archive on 21 July 2027 and serves the same endpoint.
+ *
+ * Deliberately not the default, despite being the newer archive. The wwPDB
+ * describes this host as transitional: on the cutover date the beta archive
+ * replaces the main one, after which the beta URL is supported by redirection
+ * for three years. So it is the hostname that needs changing, twice, whereas
+ * {@link #DEFAULT_SERVER_URL} becomes the new archive and needs changing never.
+ *
+ * Its value is in testing. Because this host is the post-2027 content today, a
+ * request against it checks the endpoint against the archive as it will be,
+ * rather than against the archive as it is.
+ */
+ public static final String BETA_SERVER_URL = "https://files-beta.wwpdb.org/validation/download/";
+
+ /**
+ * An EBI mirror serving byte-identical files.
+ *
+ * Unlike the two above, EBI publishes no name-resolving endpoint — only
+ * full directory paths — so selecting it means setting the divided
+ * templates as well:
+ *
+ * The endpoint resolves an entry by name, so no directory path is built here.
+ * That is deliberate. In July 2027 the archive moves to extended identifiers
+ * and a per-entry directory layout, and a path assembled from a hash and an
+ * identifier would have to be rewritten for it; a name does not. Both spellings
+ * of an identifier resolve, so whichever {@code PdbId} yields is accepted.
+ */
+ public static final String DEFAULT_TWO_FO_FC_TEMPLATE =
+ "{pdbid_lc}_validation_2fo-fc_map_coef.cif.gz";
+
+ /** Default path template for the mFo-DFc coefficients; see {@link #DEFAULT_TWO_FO_FC_TEMPLATE}. */
+ public static final String DEFAULT_FO_FC_TEMPLATE =
+ "{pdbid_lc}_validation_fo-fc_map_coef.cif.gz";
+
+ /**
+ * Path template for the 2mFo-DFc coefficients in the divided archive, for
+ * mirrors that publish directories rather than an endpoint.
+ */
+ public static final String DIVIDED_TWO_FO_FC_TEMPLATE =
+ "{mid}/{pdbid_lc}/{pdbid_lc}_validation_2fo-fc_map_coef.cif.gz";
+
+ /** Path template for the mFo-DFc coefficients in the divided archive. */
+ public static final String DIVIDED_FO_FC_TEMPLATE =
+ "{mid}/{pdbid_lc}/{pdbid_lc}_validation_fo-fc_map_coef.cif.gz";
+
+ /**
+ * Path template for the 2mFo-DFc coefficients in the per-entry archive that
+ * replaces the divided one in July 2027, relative to a base URL ending in
+ *
+ * Provided so that a mirror of the new layout can be used the day it exists,
+ * without waiting for a release.
+ */
+ public static final String ENTRIES_TWO_FO_FC_TEMPLATE =
+ "entries/{mid}/{extid}/validation_reports/{extid}_validation_2fo-fc_map_coef.cif.gz";
+
+ /** Path template for the mFo-DFc coefficients in the per-entry archive. */
+ public static final String ENTRIES_FO_FC_TEMPLATE =
+ "entries/{mid}/{extid}/validation_reports/{extid}_validation_fo-fc_map_coef.cif.gz";
+
+ private static String serverBaseUrl = DEFAULT_SERVER_URL;
+ private static String twoFoFcTemplate = DEFAULT_TWO_FO_FC_TEMPLATE;
+ private static String foFcTemplate = DEFAULT_FO_FC_TEMPLATE;
+
+ /**
+ * @param cacheRoot the BioJava cache directory
+ */
+ public WwpdbMapCoefficientsProvider(File cacheRoot) {
+ super(cacheRoot);
+ }
+
+ /** @return the base URL of the validation report archive */
+ public static String getServerBaseUrl() {
+ return serverBaseUrl;
+ }
+
+ /** @param url the base URL; a trailing slash is added if missing */
+ public static void setServerBaseUrl(String url) {
+ serverBaseUrl = url == null ? DEFAULT_SERVER_URL : (url.endsWith("/") ? url : url + "/");
+ }
+
+ /**
+ * Overrides the path template for a map kind.
+ *
+ * @param kind {@link DensityMapKind#TWO_FO_FC} or {@link DensityMapKind#FO_FC}
+ * @param template a template understood by {@link UrlTemplates}
+ */
+ public static void setPathUrlTemplate(DensityMapKind kind, String template) {
+ if (kind == DensityMapKind.TWO_FO_FC) {
+ twoFoFcTemplate = template == null ? DEFAULT_TWO_FO_FC_TEMPLATE : template;
+ } else if (kind == DensityMapKind.FO_FC) {
+ foFcTemplate = template == null ? DEFAULT_FO_FC_TEMPLATE : template;
+ } else {
+ throw new IllegalArgumentException("Map coefficients exist only for 2Fo-Fc and Fo-Fc, not " + kind);
+ }
+ }
+
+ /** Restores the default server and templates. */
+ public static void resetToDefaults() {
+ serverBaseUrl = DEFAULT_SERVER_URL;
+ twoFoFcTemplate = DEFAULT_TWO_FO_FC_TEMPLATE;
+ foFcTemplate = DEFAULT_FO_FC_TEMPLATE;
+ }
+
+ @Override
+ public DensityMapSource getSource() {
+ return DensityMapSource.WWPDB_MAP_COEFFICIENTS;
+ }
+
+ @Override
+ public DensityFileFormat getFormat() {
+ return DensityFileFormat.MAP_COEFFICIENTS_CIF_GZ;
+ }
+
+ @Override
+ public boolean supports(DensityMapKind kind) {
+ return kind == DensityMapKind.TWO_FO_FC || kind == DensityMapKind.FO_FC;
+ }
+
+ /**
+ * Builds the URL for a set of coefficients without fetching them.
+ *
+ * @param pdbId the entry
+ * @param kind the kind of map
+ * @return the URL as a string
+ */
+ public String buildUrl(PdbId pdbId, DensityMapKind kind) {
+ String template = kind == DensityMapKind.FO_FC ? foFcTemplate : twoFoFcTemplate;
+ return serverBaseUrl + UrlTemplates.expand(template, UrlTemplates.values(urlId(pdbId), null, -1));
+ }
+
+ @Override
+ public DensityMapResult fetch(DensityMapRequest request) throws IOException {
+ if (request.getPdbId() == null || !supports(request.getKind())) {
+ return null;
+ }
+ URL url = new URL(buildUrl(request.getPdbId(), request.getKind()));
+ File target = DensityCacheLayout.pdbMapFile(effectiveCacheRoot(request), request.getPdbId(),
+ request.getKind(), getSource(), getFormat(), null);
+ return obtain(request, url, target, request.getKind(), null, null, null);
+ }
+}
diff --git a/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestCcp4Header.java b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestCcp4Header.java
new file mode 100644
index 0000000000..3c5d25950c
--- /dev/null
+++ b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestCcp4Header.java
@@ -0,0 +1,112 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.zip.GZIPOutputStream;
+
+import org.biojava.nbio.core.util.FileDownloadUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The CCP4 header check that keeps a server's error page out of the cache.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class TestCcp4Header {
+
+ private File dir;
+
+ @BeforeEach
+ public void setUp() throws IOException {
+ dir = Files.createTempDirectory("bj-ccp4").toFile();
+ }
+
+ @AfterEach
+ public void tearDown() throws IOException {
+ FileDownloadUtils.deleteDirectory(dir.toPath());
+ }
+
+ /** A minimal file carrying the stamp at the offset a real CCP4 header uses. */
+ private static byte[] fakeMap() {
+ byte[] bytes = new byte[2048];
+ byte[] stamp = Ccp4Header.MAP_STAMP.getBytes(StandardCharsets.US_ASCII);
+ System.arraycopy(stamp, 0, bytes, Ccp4Header.MAP_STAMP_OFFSET, stamp.length);
+ return bytes;
+ }
+
+ private File write(String name, byte[] content) throws IOException {
+ File f = new File(dir, name);
+ Files.write(f.toPath(), content);
+ return f;
+ }
+
+ @Test
+ public void recognisesAMapByItsStamp() throws IOException {
+ assertTrue(Ccp4Header.isCcp4(write("good.ccp4", fakeMap())));
+ }
+
+ @Test
+ public void recognisesAGzippedMap() throws IOException {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+ try (GZIPOutputStream gz = new GZIPOutputStream(buffer)) {
+ gz.write(fakeMap());
+ }
+ assertTrue(Ccp4Header.isCcp4(write("good.map.gz", buffer.toByteArray())), "EMDB serves its maps gzipped");
+ }
+
+ /**
+ * The case this check exists for: a server answering with an error page and an
+ * HTTP 200, which nothing else would catch.
+ */
+ @Test
+ public void rejectsAnHtmlErrorPage() throws IOException {
+ StringBuilder html = new StringBuilder(" The requested resource was not found on this server.
+ * The behaviour that matters most here is the difference between "this source
+ * has nothing for the entry" and "this source could not be reached". The first
+ * must move on to the next source; the second must abort, because reporting a
+ * network outage as "no density exists" would be actively misleading.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class TestDensityFallbackChain {
+
+ private DensityMapCache cache;
+ private List
+ * EMD-0262 is about 116 MB, which sits comfortably under the 256 MiB default
+ * ceiling: left to the ceiling alone it would be downloaded in full. What
+ * avoids that is putting the density servers ahead of the archive, which serve
+ * a few megabytes for the same entry. The ceiling is the backstop for the maps
+ * that run to gigabytes.
+ */
+ @Test
+ public void theFullEmMapIsLargeButWithinTheDefaultCeiling() throws IOException {
+ copyResource("emdb-map-EMD-0262.json",
+ DensityCacheLayout.emdbMapInfoFile(cacheRoot, "EMD-0262"));
+ EmdbEntryResolver resolver = new EmdbEntryResolver(cacheRoot);
+ resolver.setFetchBehavior(FetchBehavior.LOCAL_ONLY);
+
+ long bytes = resolver.getEntryInfo("EMD-0262").getMapSizeBytes();
+ assertTrue(bytes > 100L * 1024 * 1024, "expected a map of order 100 MB, got " + bytes);
+ assertTrue(bytes < DensityMapCache.DEFAULT_MAX_DOWNLOAD_BYTES,
+ "the ceiling alone would not stop this download");
+
+ DensityMapCache cache = new DensityMapCache(cacheRoot.getAbsolutePath());
+ List
+ * isosurface ID "bj_density_2fofc" delete
+ *
+ * to remove just the blue surface. Pressing Reset Display keeps the maps, since
+ * they are folded into the saved state when they are drawn.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class DemoShowElectronDensity {
+
+ /**
+ * @param args an optional PDB ID to display instead of the default
+ * @throws Exception if the structure or the map could not be fetched
+ */
+ public static void main(String[] args) throws Exception {
+ String id = args.length > 0 ? args[0] : "1cbs";
+
+ Structure structure = StructureIO.getStructure(id);
+ StructureAlignmentJmol viewer = new StructureAlignmentJmol();
+ viewer.setStructure(structure);
+ viewer.evalString("select all; cartoon on; color chain; "
+ + "select ligand; wireframe 0.16; spacefill 0.4; color cpk;");
+
+ DensityMapCache cache = new DensityMapCache();
+ System.out.println("Density cache: " + cache.getCachePath());
+
+ // Clipping to the ligand keeps the surface readable and, for a large map,
+ // keeps the contouring quick enough not to stall the interface.
+ for (DensityMapKind kind : new DensityMapKind[] {DensityMapKind.TWO_FO_FC, DensityMapKind.FO_FC}) {
+ cache.findDensityMap(new PdbId(id), kind).ifPresent(map -> {
+ System.out.printf("%-8s from %-20s %s (%,d bytes)%n",
+ map.getKind(), map.getSource(), map.getFile().getName(), map.getFileSizeBytes());
+ viewer.getJmolPanel().loadDensityMap(map, "{ligand}", 5.0);
+ });
+ }
+
+ DensityMapResult any = cache.findDensityMap(new PdbId(id), DensityMapKind.AUTO).orElse(null);
+ if (any == null) {
+ System.out.println("No density is available for " + id
+ + " - try an entry with deposited structure factors, such as 1cbs.");
+ }
+ }
+}
diff --git a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MenuCreator.java b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MenuCreator.java
index 6a0e3c6e1a..197802a2c5 100644
--- a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MenuCreator.java
+++ b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MenuCreator.java
@@ -66,6 +66,8 @@ public class MenuCreator {
public static final String PAIRWISE_ALIGN = "New Pairwise Alignment";
public static final String MULTIPLE_ALIGN = "New Multiple Alignment";
public static final String PHYLOGENETIC_TREE = "Phylogenetic Tree";
+ /** @since 7.3.0 */
+ public static final String SHOW_DENSITY = "Show Electron Density";
protected static final int keyMask =
Toolkit.getDefaultToolkit().getMenuShortcutKeyMask();
@@ -169,6 +171,8 @@ public static JMenuBar initJmolMenu(JFrame frame,
distMax.setMnemonic(KeyEvent.VK_D);
distMax.addActionListener(new MyDistMaxListener(parent));
view.add(distMax);
+ //Electron density
+ view.add(getShowDensityMenuItem(parent));
//Dot Plot - only if the alignment was an afpChain
if (afpChain != null){
JMenuItem dotplot = new JMenuItem(DOT_PLOT);
@@ -229,6 +233,22 @@ public static JMenuItem getOpenPDBMenuItem() {
return openI;
}
+ /**
+ * Menu item that fetches and displays the electron density or cryo-EM map for
+ * the structure currently on screen.
+ *
+ * @param parent the viewer to draw the map into
+ * @return the menu item
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+ public static JMenuItem getShowDensityMenuItem(AbstractAlignmentJmol parent) {
+ JMenuItem densityI = new JMenuItem(SHOW_DENSITY);
+ densityI.setMnemonic(KeyEvent.VK_E);
+ densityI.addActionListener(new MyShowDensityListener(parent));
+ return densityI;
+ }
+
public static JMenuItem getLoadMenuItem() {
diff --git a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MyShowDensityListener.java b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MyShowDensityListener.java
new file mode 100644
index 0000000000..b1a9cf23ff
--- /dev/null
+++ b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MyShowDensityListener.java
@@ -0,0 +1,202 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.align.gui;
+
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+import javax.swing.JOptionPane;
+import javax.swing.SwingWorker;
+
+import org.biojava.nbio.structure.PdbId;
+import org.biojava.nbio.structure.Structure;
+import org.biojava.nbio.structure.align.gui.jmol.AbstractAlignmentJmol;
+import org.biojava.nbio.structure.io.density.DensityMapCache;
+import org.biojava.nbio.structure.io.density.DensityMapKind;
+import org.biojava.nbio.structure.io.density.DensityMapRequest;
+import org.biojava.nbio.structure.io.density.DensityMapResult;
+import org.biojava.nbio.structure.io.density.DensityMapSource;
+import org.biojava.nbio.structure.io.density.NoDensityMapException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Fetches the density map for the structure on screen and draws it.
+ * , Void>() {
+
+ private NoDensityMapException missing;
+
+ @Override
+ protected List
null before the window is built
+ * @since 7.3.0
+ */
+ public JFrame getFrame() {
+ return frame;
+ }
+
+ /**
+ * Writes a short message into the viewer's status field.
+ *
+ * @param message the message; ignored if there is no status field yet
+ * @since 7.3.0
+ */
+ public void setStatus(String message) {
+ if (status != null) {
+ status.setText(message);
+ }
+ }
+
/**
* Set the title of the AlignmentJmol window.
* @param title
diff --git a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/JmolPanel.java b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/JmolPanel.java
index 23361e62ed..9108489d29 100644
--- a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/JmolPanel.java
+++ b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/JmolPanel.java
@@ -31,12 +31,14 @@
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.BufferedInputStream;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.text.DecimalFormat;
import java.util.List;
+import java.util.Locale;
import javax.swing.JComboBox;
@@ -50,6 +52,8 @@
import org.biojava.nbio.structure.domain.pdp.Domain;
import org.biojava.nbio.structure.domain.pdp.Segment;
import org.biojava.nbio.structure.gui.util.color.ColorUtils;
+import org.biojava.nbio.structure.io.density.DensityMapKind;
+import org.biojava.nbio.structure.io.density.DensityMapResult;
import org.biojava.nbio.structure.io.mmtf.MmtfActions;
import org.biojava.nbio.structure.jama.Matrix;
import org.biojava.nbio.structure.scop.ScopDatabase;
@@ -166,6 +170,157 @@ public void setStructure(final Structure s) {
setStructure(s, false);
}
+ /** Isosurface id used for the 2mFo-DFc map, so that it can be addressed on its own. */
+ public static final String ISOSURFACE_ID_2FOFC = "bj_density_2fofc";
+
+ /** Isosurface id used for the mFo-DFc difference map, drawn as a signed pair of lobes. */
+ public static final String ISOSURFACE_ID_FOFC = "bj_density_fofc";
+
+ /** Isosurface id used for a cryo-EM map. */
+ public static final String ISOSURFACE_ID_EM = "bj_density_em";
+
+ /** Default clipping radius, in Angstroms, around the selected atoms. */
+ public static final double DEFAULT_WITHIN_RADIUS = 5.0;
+
+ /**
+ * Displays a density map fetched through
+ * {@link org.biojava.nbio.structure.io.density.DensityMapCache}, clipped to
+ * {@value #DEFAULT_WITHIN_RADIUS} Angstroms around the whole model.
+ *
+ * @param map the map to display
+ * @throws IllegalArgumentException if the map cannot be displayed without a
+ * Fourier transform first
+ * @since 7.3.0
+ */
+ public void loadDensityMap(DensityMapResult map) {
+ loadDensityMap(map, "{*}", DEFAULT_WITHIN_RADIUS);
+ }
+
+ /**
+ * Displays a density map, clipped to a distance around a selection.
+ * {*} or
+ * {ligand}, or null to contour the whole cell
+ * @param withinRadius the clipping radius in Angstroms, ignored when
+ * atomSelection is null
+ * @throws IllegalArgumentException if the map cannot be displayed without a
+ * Fourier transform first
+ * @since 7.3.0
+ */
+ public void loadDensityMap(DensityMapResult map, String atomSelection, double withinRadius) {
+ if (!map.isRenderable()) {
+ throw new IllegalArgumentException("A " + map.getFormat() + " file holds structure factors, not a "
+ + "sampled map, and cannot be displayed as it stands. Convert it first, for example with "
+ + "'gemmi sf2map', or fetch the map from a source that serves a grid.");
+ }
+
+ Double level = map.getRecommendedContourLevel();
+ if (map.getKind() == DensityMapKind.EM && level != null) {
+ // EM maps are conventionally contoured at the absolute level the
+ // depositors chose rather than at a multiple of sigma.
+ loadDensityMap(map.getFile(), map.getKind(), level, false, atomSelection, withinRadius);
+ } else {
+ double sigma = map.getKind() == DensityMapKind.TWO_FO_FC ? 1.0 : 3.0;
+ loadDensityMap(map.getFile(), map.getKind(), sigma, true, atomSelection, withinRadius);
+ }
+ }
+
+ /**
+ * Displays a density map file directly.
+ *
+ * @param mapFile the map file, in a format Jmol can contour (CCP4/MRC, or a
+ * BinaryCIF volume)
+ * @param kind what the map values mean, which decides the colouring and whether
+ * a signed pair of surfaces is drawn
+ * @param level the contour level
+ * @param levelIsSigma whether level is a multiple of the map's RMS
+ * deviation rather than an absolute value
+ * @param atomSelection a Jmol atom expression, or null to contour
+ * the whole cell
+ * @param withinRadius the clipping radius in Angstroms
+ * @since 7.3.0
+ */
+ public void loadDensityMap(File mapFile, DensityMapKind kind, double level, boolean levelIsSigma,
+ String atomSelection, double withinRadius) {
+
+ String url = toJmolFileUrl(mapFile);
+ String within = atomSelection == null ? ""
+ : String.format(Locale.US, " within %.1f %s", withinRadius, atomSelection);
+
+ if (kind == DensityMapKind.FO_FC) {
+ // One signed surface carrying both lobes, red for negative and green for
+ // positive. Drawing the negative lobe as a separate surface at "sigma -3"
+ // does not work: Jmol gives a negative sigma its own internal meaning and
+ // silently contours at the default level instead.
+ evalString(isosurfaceCommand(ISOSURFACE_ID_FOFC, "sign red green", level, levelIsSigma, within, url));
+ } else if (kind == DensityMapKind.EM) {
+ evalString(isosurfaceCommand(ISOSURFACE_ID_EM, "color grey", level, levelIsSigma, within, url));
+ } else {
+ evalString(isosurfaceCommand(ISOSURFACE_ID_2FOFC, "color blue", level, levelIsSigma, within, url));
+ }
+
+ // resetDisplay() restores "state_1", which is saved when the structure is
+ // loaded. Without re-saving here, pressing Reset Display would silently
+ // discard the map the user just asked for.
+ evalString("save STATE state_1");
+ }
+
+ /**
+ * Builds an isosurface command.
+ * mesh and
+ * nofill have to follow the file name. Placed before it, Jmol
+ * accepts the command without complaint and draws nothing at all.
+ */
+ private static String isosurfaceCommand(String id, String colouring, double level, boolean levelIsSigma,
+ String within, String url) {
+ return String.format(Locale.US,
+ "isosurface ID \"%s\" delete; isosurface ID \"%s\" %s %s %.4f%s \"%s\" mesh nofill;",
+ id, id, colouring, levelIsSigma ? "sigma" : "cutoff", level, within, url);
+ }
+
+ /**
+ * Removes any density surfaces this panel has drawn, leaving other isosurfaces
+ * alone.
+ *
+ * @since 7.3.0
+ */
+ public void clearDensityMaps() {
+ for (String id : new String[] {ISOSURFACE_ID_2FOFC, ISOSURFACE_ID_FOFC, ISOSURFACE_ID_EM}) {
+ evalString("isosurface ID \"" + id + "\" delete;");
+ }
+ evalString("save STATE state_1");
+ }
+
+ /**
+ * Converts a file to the URL form Jmol expects.
+ * file:/C:/...,
+ * which is normalised here to the usual three-slash form.
+ *
+ * @param file the file
+ * @return a URL string safe to embed in a Jmol script
+ * @since 7.3.0
+ */
+ public static String toJmolFileUrl(File file) {
+ String url = file.getAbsoluteFile().toURI().toString();
+ if (url.startsWith("file:/") && !url.startsWith("file://")) {
+ url = "file:///" + url.substring("file:/".length());
+ }
+ return url;
+ }
+
/** assign a custom color to the Jmol chains command.
*
*/
diff --git a/biojava-structure/src/main/java/demo/DemoFetchElectronDensity.java b/biojava-structure/src/main/java/demo/DemoFetchElectronDensity.java
new file mode 100644
index 0000000000..f987fd7cfe
--- /dev/null
+++ b/biojava-structure/src/main/java/demo/DemoFetchElectronDensity.java
@@ -0,0 +1,90 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package demo;
+
+import java.io.IOException;
+
+import org.biojava.nbio.structure.PdbId;
+import org.biojava.nbio.structure.io.density.DensityMapCache;
+import org.biojava.nbio.structure.io.density.DensityMapKind;
+import org.biojava.nbio.structure.io.density.DensityMapResult;
+import org.biojava.nbio.structure.io.density.NoDensityMapException;
+
+/**
+ * Fetches electron density and cryo-EM maps, printing which source answered.
+ *
+ *
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class DemoFetchElectronDensity {
+
+ /**
+ * @param args ignored
+ * @throws IOException if a server could not be reached at all
+ */
+ public static void main(String[] args) throws IOException {
+ DensityMapCache cache = new DensityMapCache();
+ System.out.println("Caching under: " + cache.getCachePath());
+ System.out.println();
+
+ show(cache, "1cbs", DensityMapKind.TWO_FO_FC);
+ show(cache, "1cbs", DensityMapKind.FO_FC);
+ show(cache, "6hu9", DensityMapKind.AUTO);
+ show(cache, "4hhb", DensityMapKind.AUTO);
+ }
+
+ private static void show(DensityMapCache cache, String id, DensityMapKind kind) throws IOException {
+ System.out.printf("%s (%s)%n", id, kind);
+ try {
+ DensityMapResult result = cache.getDensityMap(new PdbId(id), kind);
+ System.out.printf(" source : %s%n", result.getSource());
+ System.out.printf(" format : %s%s%n", result.getFormat(),
+ result.isRenderable() ? "" : " (needs an FFT before display)");
+ System.out.printf(" kind : %s%n", result.getKind());
+ System.out.printf(" file : %s (%,d bytes)%n", result.getFile(), result.getFileSizeBytes());
+ System.out.printf(" cached : %s%n", result.isFromCache());
+ if (result.getEmdbId() != null) {
+ System.out.printf(" EMDB : %s%n", result.getEmdbId());
+ }
+ if (result.getRecommendedContourLevel() != null) {
+ System.out.printf(" contour : %s (author recommended)%n", result.getRecommendedContourLevel());
+ }
+ if (result.getContourInSigma() != null) {
+ System.out.printf(" in sigma : %.2f%n", result.getContourInSigma());
+ }
+ } catch (NoDensityMapException e) {
+ System.out.printf(" no map available%n");
+ e.getAttempts().forEach((source, reason) ->
+ System.out.printf(" %-24s %s%n", source, reason));
+ }
+ System.out.println();
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/AbstractDensityMapProvider.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/AbstractDensityMapProvider.java
new file mode 100644
index 0000000000..3ac4057ba2
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/AbstractDensityMapProvider.java
@@ -0,0 +1,360 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Date;
+
+import org.biojava.nbio.core.util.FileDownloadUtils;
+import org.biojava.nbio.core.util.HttpStatusException;
+import org.biojava.nbio.structure.PdbId;
+import org.biojava.nbio.structure.io.LocalPDBDirectory;
+import org.biojava.nbio.structure.io.LocalPDBDirectory.FetchBehavior;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Shared machinery for the concrete density map providers: cache lookup,
+ * download with validation, size limiting and format sanity checking.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public abstract class AbstractDensityMapProvider implements DensityMapProvider {
+
+ private static final Logger logger = LoggerFactory.getLogger(AbstractDensityMapProvider.class);
+
+ /**
+ * Anything smaller than this is not a usable map. Density servers answer with a
+ * short body rather than an error status when they have nothing, so a size
+ * floor is a necessary part of deciding whether a download succeeded.
+ */
+ public static final long MIN_DENSITY_FILE_SIZE = 1024L;
+
+ private File cacheRoot;
+ private FetchBehavior fetchBehavior = FetchBehavior.FETCH_FILES;
+ private long maxDownloadBytes = DensityMapCache.DEFAULT_MAX_DOWNLOAD_BYTES;
+
+ /**
+ * @param cacheRoot the BioJava cache directory that the density directory sits in
+ */
+ protected AbstractDensityMapProvider(File cacheRoot) {
+ this.cacheRoot = cacheRoot;
+ }
+
+ /** @return the BioJava cache directory */
+ public File getCacheRoot() {
+ return cacheRoot;
+ }
+
+ /** @param cacheRoot the BioJava cache directory */
+ public void setCacheRoot(File cacheRoot) {
+ this.cacheRoot = cacheRoot;
+ }
+
+ /** @return how aggressively this provider re-fetches */
+ public FetchBehavior getFetchBehavior() {
+ return fetchBehavior;
+ }
+
+ /** @param fetchBehavior how aggressively to re-fetch */
+ public void setFetchBehavior(FetchBehavior fetchBehavior) {
+ this.fetchBehavior = fetchBehavior == null ? FetchBehavior.FETCH_FILES : fetchBehavior;
+ }
+
+ /** @return the download size limit in bytes, or 0 for no limit */
+ public long getMaxDownloadBytes() {
+ return maxDownloadBytes;
+ }
+
+ /** @param maxDownloadBytes the download size limit in bytes, or 0 for no limit */
+ public void setMaxDownloadBytes(long maxDownloadBytes) {
+ this.maxDownloadBytes = maxDownloadBytes;
+ }
+
+ @Override
+ public boolean supports(DensityMapKind kind) {
+ return kind != null && kind != DensityMapKind.AUTO;
+ }
+
+ /**
+ * The identifier spelling to put into a URL: four characters and lower case
+ * where the entry has a short form.
+ *
+ * @param pdbId the entry
+ * @return the identifier for use in a URL
+ */
+ protected String urlId(PdbId pdbId) {
+ return DensityCacheLayout.shortIdOrFull(pdbId).toLowerCase();
+ }
+
+ /**
+ * The effective fetch behaviour for a request, preferring the request's own
+ * setting over this provider's.
+ *
+ * @param request the request
+ * @return the behaviour to apply
+ */
+ protected FetchBehavior effectiveFetchBehavior(DensityMapRequest request) {
+ return request.getFetchBehavior() == null ? fetchBehavior : request.getFetchBehavior();
+ }
+
+ /**
+ * The effective size limit for a request, preferring the request's own setting.
+ *
+ * @param request the request
+ * @return the limit in bytes, or 0 for no limit
+ */
+ protected long effectiveMaxBytes(DensityMapRequest request) {
+ return request.getMaxDownloadBytes() < 0 ? maxDownloadBytes : request.getMaxDownloadBytes();
+ }
+
+ /**
+ * The cache directory to use for a request, preferring the request's override.
+ *
+ * @param request the request
+ * @return the cache directory
+ */
+ protected File effectiveCacheRoot(DensityMapRequest request) {
+ return request.getCacheDir() == null ? cacheRoot : request.getCacheDir();
+ }
+
+ /**
+ * Obtains a map, serving it from the cache when the fetch behaviour permits and
+ * downloading it otherwise.
+ *
+ * @param request what was asked for
+ * @param url where to fetch from
+ * @param target where to cache it
+ * @param kind the concrete kind of map being fetched
+ * @param emdbId the EMDB entry, if this is an EM map; otherwise null
+ * @param contourLevel the author-recommended contour level, or null
+ * @param sigma the map RMS deviation, or null
+ * @return the result, or null if the source has nothing for this entry
+ * @throws IOException on transport failure
+ */
+ protected DensityMapResult obtain(DensityMapRequest request, URL url, File target, DensityMapKind kind,
+ String emdbId, Double contourLevel, Double sigma) throws IOException {
+
+ FetchBehavior behavior = effectiveFetchBehavior(request);
+
+ if (isCacheUsable(target, url, behavior)) {
+ // The kind always comes from the request, never from the sidecar: a single
+ // cached file can hold more than one kind of map, so the sidecar's kind
+ // records what was asked for first, not what the caller wants now.
+ DensityMapResult cached = DensityMapResult.readMeta(target);
+ Double cachedContour = contourLevel != null || cached == null ? contourLevel
+ : cached.getRecommendedContourLevel();
+ Double cachedSigma = sigma != null || cached == null ? sigma : cached.getSigma();
+ DensityMapResult result = new DensityMapResult(target, getSource(), getFormat(), kind,
+ request.getPdbId(), emdbId, url.toString(), true, cachedContour, cachedSigma);
+ if (cached == null) {
+ // The file is good but its description was lost; write one rather than
+ // downloading several megabytes again.
+ result.writeMeta();
+ }
+ return result;
+ }
+
+ if (behavior == FetchBehavior.LOCAL_ONLY) {
+ throw new HttpStatusException(404, url.toString(),
+ "not in the local cache and downloads are disabled (FetchBehavior.LOCAL_ONLY)");
+ }
+
+ File dir = target.getAbsoluteFile().getParentFile();
+ if (!dir.isDirectory() && !dir.mkdirs()) {
+ throw new IOException("Could not create density cache directory " + dir);
+ }
+
+ enforceSizeLimit(url, effectiveMaxBytes(request), request);
+
+ logger.info("Fetching {} density map for {} from {}", kind,
+ request.getPdbId() == null ? emdbId : request.getPdbId().getId(), url);
+ FileDownloadUtils.downloadFileWithValidation(url, target, null, FileDownloadUtils.Hash.UNKNOWN,
+ FileDownloadUtils.ETagPolicy.USE_IF_HEX_DIGEST);
+
+ if (!isPlausibleMap(target)) {
+ // Do not leave a bad file behind to be picked up as a cache hit later.
+ deleteWithSidecars(target);
+ throw new IOException("The content downloaded from " + url + " is not a usable "
+ + getFormat() + " density map.");
+ }
+
+ DensityMapResult result = new DensityMapResult(target, getSource(), getFormat(), kind,
+ request.getPdbId(), emdbId, url.toString(), false, contourLevel, sigma);
+ result.writeMeta();
+ return result;
+ }
+
+ /**
+ * Whether a cached file may be used as-is.
+ *
+ * @param target the cached file
+ * @param url where it came from
+ * @param behavior the fetch behaviour in force
+ * @return true if the cached file should be served
+ */
+ protected boolean isCacheUsable(File target, URL url, FetchBehavior behavior) {
+ if (behavior == FetchBehavior.FORCE_DOWNLOAD) {
+ return false;
+ }
+ if (!target.isFile() || target.length() < MIN_DENSITY_FILE_SIZE) {
+ return false;
+ }
+ if (!FileDownloadUtils.validateFile(target)) {
+ logger.info("Cached density map [{}] failed validation and will be re-downloaded.", target);
+ return false;
+ }
+ if (!isPlausibleMap(target)) {
+ logger.info("Cached file [{}] is not a usable {} map and will be re-downloaded.", target, getFormat());
+ return false;
+ }
+ if (behavior == FetchBehavior.LOCAL_ONLY) {
+ return true;
+ }
+ if (behavior == FetchBehavior.FETCH_IF_OUTDATED) {
+ Date serverDate = LocalPDBDirectory.getLastModifiedTime(url);
+ if (serverDate == null) {
+ // Density servers generate their responses on the fly and send no
+ // Last-Modified header. Treating that as "outdated" would re-download
+ // on every single call, so an unknown timestamp keeps the cache.
+ logger.debug("No server timestamp for {}; keeping the cached copy.", url);
+ return true;
+ }
+ return target.lastModified() >= serverDate.getTime();
+ }
+ // FETCH_FILES, and FETCH_REMEDIATED which has no meaning for maps.
+ return true;
+ }
+
+ /**
+ * Checks that a file looks like the format this provider delivers. Only the
+ * CCP4/MRC formats carry a recognisable stamp; the others are accepted on size
+ * alone.
+ *
+ * @param file the file to check
+ * @return true if the file is plausibly a map of this format
+ */
+ protected boolean isPlausibleMap(File file) {
+ if (file == null || file.length() < MIN_DENSITY_FILE_SIZE) {
+ return false;
+ }
+ DensityFileFormat format = getFormat();
+ if (format == DensityFileFormat.CCP4 || format == DensityFileFormat.CCP4_GZ) {
+ return Ccp4Header.isCcp4Quietly(file);
+ }
+ return true;
+ }
+
+ /**
+ * Refuses a download whose declared size exceeds the limit, before any of the
+ * body is transferred.
+ *
+ * @param url the resource
+ * @param maxBytes the limit, or 0 for no limit
+ * @param request the request being served, used for reporting
+ * @throws DensityMapTooLargeException if the resource is too large
+ */
+ protected void enforceSizeLimit(URL url, long maxBytes, DensityMapRequest request)
+ throws DensityMapTooLargeException {
+ if (maxBytes <= 0) {
+ return;
+ }
+ long size = declaredSize(url);
+ if (size > maxBytes) {
+ reportTooLarge(url, size, maxBytes, request);
+ throw new DensityMapTooLargeException(url.toString(), size, maxBytes);
+ }
+ }
+
+ /**
+ * Announces that a map was skipped for being too large.
+ * Content-Length a server declares for a resource.
+ *
+ * @param url the resource
+ * @return the size in bytes, or a negative value if the server did not say
+ */
+ protected long declaredSize(URL url) {
+ try {
+ URLConnection connection = FileDownloadUtils.prepareURLConnection(url.toString(), 30000);
+ if (connection instanceof java.net.HttpURLConnection) {
+ ((java.net.HttpURLConnection) connection).setRequestMethod("HEAD");
+ }
+ connection.connect();
+ try {
+ return connection.getContentLengthLong();
+ } finally {
+ if (connection instanceof java.net.HttpURLConnection) {
+ ((java.net.HttpURLConnection) connection).disconnect();
+ }
+ }
+ } catch (IOException e) {
+ logger.debug("Could not determine the size of {}: {}", url, e.getMessage());
+ return -1;
+ }
+ }
+
+ /**
+ * Deletes a cached map along with its validation and metadata sidecars.
+ *
+ * @param target the cached map
+ */
+ protected void deleteWithSidecars(File target) {
+ File dir = target.getAbsoluteFile().getParentFile();
+ if (dir == null) {
+ return;
+ }
+ File[] siblings = dir.listFiles((d, name) -> name.startsWith(target.getName()));
+ if (siblings == null) {
+ return;
+ }
+ for (File f : siblings) {
+ if (!f.delete()) {
+ logger.debug("Could not delete [{}]", f);
+ }
+ }
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/Ccp4Header.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/Ccp4Header.java
new file mode 100644
index 0000000000..594ad91927
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/Ccp4Header.java
@@ -0,0 +1,133 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.zip.GZIPInputStream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Recognises CCP4/MRC map files by their header.
+ * MAP
+ * stamp does, turning a silent failure into a clean cache miss.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class Ccp4Header {
+
+ private static final Logger logger = LoggerFactory.getLogger(Ccp4Header.class);
+
+ /**
+ * Byte offset of the four-character format stamp within a CCP4/MRC header. It
+ * sits in word 53 of the 256-word header.
+ */
+ public static final int MAP_STAMP_OFFSET = 208;
+
+ /** The stamp itself: the three letters of "MAP" followed by a space. */
+ public static final String MAP_STAMP = "MAP ";
+
+ /** Number of header bytes that must be readable for the check to be possible. */
+ private static final int HEADER_BYTES = MAP_STAMP_OFFSET + 4;
+
+ private Ccp4Header() {
+ }
+
+ /**
+ * Checks whether a file is a CCP4/MRC map. Gzipped files are decompressed on
+ * the fly, so .map.gz works as well as .ccp4.
+ *
+ * @param file the file to check
+ * @return true if the CCP4 stamp is present
+ * @throws IOException if the file could not be read
+ */
+ public static boolean isCcp4(File file) throws IOException {
+ if (file == null || !file.isFile()) {
+ return false;
+ }
+ // Deliberately no shortcut on file.length(): a gzipped map compresses to far
+ // less than the size of the header it contains, so a length test here would
+ // reject perfectly good small maps. Reading decides it instead.
+ try (InputStream in = openPossiblyGzipped(file)) {
+ return isCcp4(in);
+ }
+ }
+
+ /**
+ * Checks whether a stream carries a CCP4/MRC map header. The stream is read
+ * from its current position and is not closed; it is not un-read afterwards,
+ * so pass a fresh stream or one that supports marking.
+ *
+ * @param in the stream to check, already decompressed
+ * @return true if the CCP4 stamp is present
+ * @throws IOException if the stream could not be read
+ */
+ public static boolean isCcp4(InputStream in) throws IOException {
+ byte[] header = new byte[HEADER_BYTES];
+ int read = 0;
+ while (read < HEADER_BYTES) {
+ int n = in.read(header, read, HEADER_BYTES - read);
+ if (n < 0) {
+ return false; // shorter than a CCP4 header, so certainly not one
+ }
+ read += n;
+ }
+ String stamp = new String(header, MAP_STAMP_OFFSET, 4, StandardCharsets.US_ASCII);
+ return MAP_STAMP.equals(stamp);
+ }
+
+ /**
+ * Same as {@link #isCcp4(File)} but reports a problem rather than propagating
+ * it, for use in cache-validity checks where an unreadable file and an invalid
+ * one lead to the same action.
+ *
+ * @param file the file to check
+ * @return true if the file is readable and carries the CCP4 stamp
+ */
+ public static boolean isCcp4Quietly(File file) {
+ try {
+ return isCcp4(file);
+ } catch (IOException e) {
+ logger.debug("Could not read [{}] to check for a CCP4 header: {}", file, e.getMessage());
+ return false;
+ }
+ }
+
+ private static InputStream openPossiblyGzipped(File file) throws IOException {
+ InputStream in = new BufferedInputStream(Files.newInputStream(file.toPath()));
+ in.mark(2);
+ int b1 = in.read();
+ int b2 = in.read();
+ in.reset();
+ if (b1 == 0x1F && b2 == 0x8B) {
+ return new GZIPInputStream(in);
+ }
+ return in;
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityCacheLayout.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityCacheLayout.java
new file mode 100644
index 0000000000..60e6ae7b61
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityCacheLayout.java
@@ -0,0 +1,242 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.io.File;
+
+import org.biojava.nbio.structure.PdbId;
+import org.biojava.nbio.structure.io.LocalPDBDirectory;
+
+/**
+ * Where cached density files live on disk.
+ *
+ * <cache>/density/cb/1cbs_2fofc_pdbe.ccp4
+ * <cache>/density/cb/1cbs_fofc_pdbe.ccp4
+ * <cache>/density/cb/1cbs_2fofc_rcsbvs_d0.bcif
+ * <cache>/density/cb/1cbs_2fofc_wwpdb.cif.gz
+ *
+ * Both the kind and the source appear in the file name, so no two combinations
+ * can collide.
+ *
+ * <cache>/density/emd/EMD-0262/emd_0262.map.gz
+ *
+ * Several PDB entries are often fitted into a single EM map, and those maps can
+ * be hundreds of megabytes, so keying them by PDB entry would cache the same
+ * enormous file many times over.
+ * 1cbs and pdb_00001cbs both land
+ * in cb — which is also the rule the wwPDB documents for the
+ * new archive.
+ * rsync --delete. Should
+ * that judgement ever be revisited, every cached path is computed here and
+ * nowhere else, so a different layout is a change to this class plus a fallback
+ * probe for files in the old places — not a cache that everyone has to
+ * discard.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class DensityCacheLayout {
+
+ /** Name of the density sub-directory within the BioJava cache directory. */
+ public static final String DENSITY_DIR = "density";
+
+ /** Name of the sub-directory holding EMDB-keyed maps. */
+ public static final String EMDB_DIR = "emd";
+
+ /** Name of the sub-directory holding cached PDB-to-EMDB mappings. */
+ public static final String EMDB_MAPPING_DIR = "emdb-mapping";
+
+ /**
+ * File-name token for a file that holds more than one kind of map, as a density
+ * server response does.
+ */
+ public static final String BOTH_KINDS_TOKEN = "both";
+
+ private DensityCacheLayout() {
+ }
+
+ /**
+ * The root density directory inside a cache directory.
+ *
+ * @param cacheRoot the BioJava cache directory
+ * @return the density directory, which need not exist yet
+ */
+ public static File densityRoot(File cacheRoot) {
+ return new File(cacheRoot, DENSITY_DIR);
+ }
+
+ /**
+ * The cache file for a PDB-keyed map.
+ *
+ * @param cacheRoot the BioJava cache directory
+ * @param pdbId the entry
+ * @param kind the kind of map
+ * @param source the service it came from
+ * @param format the file format
+ * @param qualifier an extra discriminator such as a detail level, or
+ * null. Anything that changes the content but not the
+ * entry, kind or source belongs here.
+ * @return the file, which need not exist
+ */
+ public static File pdbMapFile(File cacheRoot, PdbId pdbId, DensityMapKind kind, DensityMapSource source,
+ DensityFileFormat format, String qualifier) {
+ return pdbMapFile(cacheRoot, pdbId, kind.getFileToken(), source, format, qualifier);
+ }
+
+ /**
+ * The cache file for a PDB-keyed map, naming the kind explicitly.
+ * null
+ * @return the file, which need not exist
+ */
+ public static File pdbMapFile(File cacheRoot, PdbId pdbId, String kindToken, DensityMapSource source,
+ DensityFileFormat format, String qualifier) {
+ String id = shortIdOrFull(pdbId).toLowerCase();
+ File dir = new File(densityRoot(cacheRoot), LocalPDBDirectory.getMiddleHash(id));
+ StringBuilder name = new StringBuilder(id)
+ .append('_').append(kindToken)
+ .append('_').append(source.getFileToken());
+ if (qualifier != null && !qualifier.isEmpty()) {
+ name.append('_').append(qualifier);
+ }
+ name.append(format.getExtension());
+ return new File(dir, name.toString());
+ }
+
+ /**
+ * The cache file for an EMDB-keyed map.
+ *
+ * @param cacheRoot the BioJava cache directory
+ * @param emdbId the EMDB entry, in any accepted form
+ * @param source the service it came from
+ * @param format the file format
+ * @param qualifier an extra discriminator such as a detail level, or null
+ * @return the file, which need not exist
+ */
+ public static File emdbMapFile(File cacheRoot, String emdbId, DensityMapSource source,
+ DensityFileFormat format, String qualifier) {
+ String canonical = DensityMapRequest.normalizeEmdbId(emdbId);
+ String number = DensityMapRequest.emdbNumber(emdbId);
+ File dir = new File(new File(densityRoot(cacheRoot), EMDB_DIR), canonical);
+ StringBuilder name = new StringBuilder("emd_").append(number);
+ if (source != DensityMapSource.EMDB_MAP) {
+ name.append('_').append(source.getFileToken());
+ }
+ if (qualifier != null && !qualifier.isEmpty()) {
+ name.append('_').append(qualifier);
+ }
+ name.append(format.getExtension());
+ return new File(dir, name.toString());
+ }
+
+ /**
+ * The file caching the EMDB identifiers and author contour level associated
+ * with a PDB entry.
+ *
+ * @param cacheRoot the BioJava cache directory
+ * @param pdbId the entry
+ * @return the file, which need not exist
+ */
+ public static File emdbMappingFile(File cacheRoot, PdbId pdbId) {
+ String id = shortIdOrFull(pdbId).toLowerCase();
+ File dir = new File(new File(densityRoot(cacheRoot), EMDB_MAPPING_DIR), LocalPDBDirectory.getMiddleHash(id));
+ return new File(dir, id + ".emdb.properties");
+ }
+
+ /**
+ * The file caching an EMDB entry's map metadata as served by the EMDB API.
+ *
+ * @param cacheRoot the BioJava cache directory
+ * @param emdbId the EMDB entry, in any accepted form
+ * @return the file, which need not exist
+ */
+ public static File emdbMapInfoFile(File cacheRoot, String emdbId) {
+ String canonical = DensityMapRequest.normalizeEmdbId(emdbId);
+ File dir = new File(new File(densityRoot(cacheRoot), EMDB_DIR), canonical);
+ return new File(dir, canonical + ".map-info.json");
+ }
+
+ /**
+ * The companion name a density-server file must have for Jmol to read its
+ * difference-map block.
+ * 2FO-FC and an
+ * FO-FC data block, and Jmol's reader chooses between them by
+ * testing whether the file name contains the literal text
+ * &diff=1. That works for a URL fetched straight from the
+ * server, where the marker rides along in the query string, but a cached local
+ * file has no query string; appending the marker to the file URL only makes
+ * Jmol look for a file that does not exist. Putting the marker into the name
+ * itself is what actually selects the block.
+ * BCifDensityReader still carries the author's
+ * "what about cached data" to-do beside it. Should Jmol gain a cleaner way to
+ * choose the block, this can be retired.
+ *
+ * @param mapFile the cached density-server file
+ * @return the sibling path that selects the difference map
+ */
+ public static File differenceMarkerFile(File mapFile) {
+ String name = mapFile.getName();
+ int dot = name.lastIndexOf('.');
+ String stem = dot < 0 ? name : name.substring(0, dot);
+ String ext = dot < 0 ? "" : name.substring(dot);
+ return new File(mapFile.getAbsoluteFile().getParentFile(), stem + "&diff=1" + ext);
+ }
+
+ /**
+ * The short four-character spelling of an identifier where one exists.
+ * gemmi sf2map, or CCP4's fft after
+ * cif2mtz) is needed before these can be displayed.
+ */
+ MAP_COEFFICIENTS_CIF_GZ(".cif.gz", false);
+
+ private final String extension;
+ private final boolean jmolLoadable;
+
+ DensityFileFormat(String extension, boolean jmolLoadable) {
+ this.extension = extension;
+ this.jmolLoadable = jmolLoadable;
+ }
+
+ /**
+ * @return the file extension used for cached files of this format, including
+ * the leading dot
+ */
+ public String getExtension() {
+ return extension;
+ }
+
+ /**
+ * Whether a viewer can contour this file as it stands.
+ *
+ * @return false for {@link #MAP_COEFFICIENTS_CIF_GZ}, which needs
+ * an FFT first; true for the sampled grid formats
+ */
+ public boolean isJmolLoadable() {
+ return jmolLoadable;
+ }
+
+ /**
+ * Whether files of this format are gzip-compressed on disk. Jmol detects gzip
+ * from the magic bytes, so such files do not need decompressing before display.
+ *
+ * @return true for the compressed formats
+ */
+ public boolean isCompressed() {
+ return this == CCP4_GZ || this == MAP_COEFFICIENTS_CIF_GZ;
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapCache.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapCache.java
new file mode 100644
index 0000000000..adbbd917a8
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapCache.java
@@ -0,0 +1,526 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import org.biojava.nbio.core.util.FileDownloadUtils;
+import org.biojava.nbio.core.util.HttpStatusException;
+import org.biojava.nbio.structure.ExperimentalTechnique;
+import org.biojava.nbio.structure.PdbId;
+import org.biojava.nbio.structure.Structure;
+import org.biojava.nbio.structure.align.util.UserConfiguration;
+import org.biojava.nbio.structure.io.LocalPDBDirectory.FetchBehavior;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Downloads and caches electron density and cryo-EM maps, trying several sources
+ * in turn until one produces a map.
+ * PDB_CACHE_DIR),
+ * laid out as described in {@link DensityCacheLayout}. Typical use is simply:
+ *
+ * DensityMapCache cache = new DensityMapCache();
+ * DensityMapResult map = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
+ * File file = map.getFile();
+ *
+ * PDB_CACHE_DIR falling back to PDB_DIR.
+ */
+ public DensityMapCache() {
+ this(new UserConfiguration().getCacheFilePath());
+ }
+
+ /**
+ * @param cachePath the directory to cache under
+ */
+ public DensityMapCache(String cachePath) {
+ this.cacheRoot = new File(FileDownloadUtils.expandUserHome(cachePath));
+ this.emdbResolver = new EmdbEntryResolver(cacheRoot);
+ buildDefaultProviders();
+ }
+
+ /**
+ * A lazily created shared instance, for callers that do not want to manage one.
+ *
+ * @return the shared cache
+ */
+ public static synchronized DensityMapCache getInstance() {
+ if (instance == null) {
+ instance = new DensityMapCache();
+ }
+ return instance;
+ }
+
+ private void buildDefaultProviders() {
+ providers.clear();
+ register(new VolumeServerProvider(cacheRoot, VolumeServerProvider.Host.RCSB));
+ register(new VolumeServerProvider(cacheRoot, VolumeServerProvider.Host.PDBE));
+ register(new PdbeCcp4MapProvider(cacheRoot));
+ register(new EmdbMapProvider(cacheRoot, emdbResolver));
+ register(new WwpdbMapCoefficientsProvider(cacheRoot));
+ applySettingsToProviders();
+ }
+
+ private void register(DensityMapProvider provider) {
+ providers.put(provider.getSource(), provider);
+ }
+
+ private void applySettingsToProviders() {
+ for (DensityMapProvider p : providers.values()) {
+ if (p instanceof AbstractDensityMapProvider) {
+ AbstractDensityMapProvider a = (AbstractDensityMapProvider) p;
+ a.setCacheRoot(cacheRoot);
+ a.setFetchBehavior(fetchBehavior);
+ a.setMaxDownloadBytes(maxDownloadBytes);
+ }
+ }
+ emdbResolver.setCacheRoot(cacheRoot);
+ emdbResolver.setFetchBehavior(fetchBehavior);
+ }
+
+ /** @return the directory maps are cached under */
+ public String getCachePath() {
+ return cacheRoot.getAbsolutePath();
+ }
+
+ /** @param cachePath the directory to cache under */
+ public void setCachePath(String cachePath) {
+ this.cacheRoot = new File(FileDownloadUtils.expandUserHome(cachePath));
+ applySettingsToProviders();
+ }
+
+ /** @return how aggressively cached maps are re-fetched */
+ public FetchBehavior getFetchBehavior() {
+ return fetchBehavior;
+ }
+
+ /**
+ * @param fetchBehavior how aggressively to re-fetch. {@link FetchBehavior#FETCH_REMEDIATED}
+ * behaves as {@link FetchBehavior#FETCH_FILES}: the 2011 remediation date
+ * is a coordinate-file concept with no meaning for maps.
+ */
+ public void setFetchBehavior(FetchBehavior fetchBehavior) {
+ this.fetchBehavior = fetchBehavior == null ? FetchBehavior.FETCH_FILES : fetchBehavior;
+ applySettingsToProviders();
+ }
+
+ /** @return the ceiling on a single download in bytes, or 0 for no limit */
+ public long getMaxDownloadBytes() {
+ return maxDownloadBytes;
+ }
+
+ /** @param maxDownloadBytes the ceiling on a single download in bytes, or 0 for no limit */
+ public void setMaxDownloadBytes(long maxDownloadBytes) {
+ this.maxDownloadBytes = maxDownloadBytes;
+ applySettingsToProviders();
+ }
+
+ /**
+ * The order sources are tried in for a given kind of map.
+ *
+ * @param kind the kind of map
+ * @return the source order
+ */
+ public ListEMD-0262
+ * @param kind the kind of map wanted
+ * @return the map
+ * @throws NoDensityMapException if no enabled source has a map for this entry
+ * @throws IOException on transport failure
+ */
+ public DensityMapResult getDensityMap(String id, DensityMapKind kind) throws IOException {
+ return getDensityMap(DensityMapRequest.builder(id).kind(kind).build());
+ }
+
+ /**
+ * Fetches a density map for an already-loaded structure.
+ * null if it is not cached
+ */
+ public DensityMapResult getCached(PdbId pdbId, DensityMapKind kind, DensityMapSource source) {
+ DensityMapProvider provider = providers.get(source);
+ if (provider == null) {
+ return null;
+ }
+ File file = DensityCacheLayout.pdbMapFile(cacheRoot, pdbId, kind, source, provider.getFormat(), null);
+ return file.isFile() ? DensityMapResult.readMeta(file) : null;
+ }
+
+ /**
+ * Removes every cached density file for an entry, including the sidecars.
+ *
+ * @param pdbId the entry
+ * @return the number of files deleted
+ */
+ public int deleteDensityMaps(PdbId pdbId) {
+ String id = DensityCacheLayout.shortIdOrFull(pdbId).toLowerCase();
+ File dir = DensityCacheLayout.pdbMapFile(cacheRoot, pdbId, DensityMapKind.TWO_FO_FC,
+ DensityMapSource.PDBE_CCP4, DensityFileFormat.CCP4, null).getParentFile();
+ File[] files = dir == null ? null : dir.listFiles((d, name) -> name.startsWith(id + "_"));
+ if (files == null) {
+ return 0;
+ }
+ int deleted = 0;
+ for (File f : files) {
+ if (f.delete()) {
+ deleted++;
+ }
+ }
+ return deleted;
+ }
+
+ private DensityMapResult getDensityMap(DensityMapRequest request, Listnull for {@link #AUTO}, which is never
+ * itself cached
+ */
+ public String getFileToken() {
+ return fileToken;
+ }
+
+ /**
+ * Expands this kind into the concrete kinds to try, in order.
+ *
+ * @return a single-element list for a concrete kind, or the X-ray-then-EM order
+ * for {@link #AUTO}
+ */
+ public Listtrue only for {@link #FO_FC}
+ */
+ public boolean isDifferenceMap() {
+ return this == FO_FC;
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapProvider.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapProvider.java
new file mode 100644
index 0000000000..c628387ffd
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapProvider.java
@@ -0,0 +1,78 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.io.IOException;
+
+import org.biojava.nbio.core.util.HttpStatusException;
+
+/**
+ * Fetches density maps from one particular service.
+ *
+ *
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public interface DensityMapProvider {
+
+ /**
+ * @return which service this provider talks to
+ */
+ DensityMapSource getSource();
+
+ /**
+ * @return the format this provider delivers
+ */
+ DensityFileFormat getFormat();
+
+ /**
+ * Whether this provider can serve a given kind of map at all. Used to skip
+ * requests that could not possibly succeed, such as asking an X-ray map service
+ * for a cryo-EM map.
+ *
+ * @param kind the kind of map wanted
+ * @return null — when this service simply has nothing
+ * for the entry. The chain moves on to the next source.true if it is worth trying
+ */
+ boolean supports(DensityMapKind kind);
+
+ /**
+ * Fetches a map, using the cache if the request's fetch behaviour allows.
+ *
+ * @param request what is wanted. Its kind is always concrete, never
+ * {@link DensityMapKind#AUTO}.
+ * @return the map, or null if this service has nothing for the entry
+ * @throws DensityMapTooLargeException if the map exceeds the request's size limit
+ * @throws IOException on transport failure; use {@link HttpStatusException} so
+ * that a missing resource can be told apart from a broken connection
+ */
+ DensityMapResult fetch(DensityMapRequest request) throws IOException;
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapRequest.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapRequest.java
new file mode 100644
index 0000000000..d6677a3835
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapRequest.java
@@ -0,0 +1,275 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.List;
+
+import org.biojava.nbio.structure.PdbId;
+import org.biojava.nbio.structure.io.LocalPDBDirectory.FetchBehavior;
+
+/**
+ * A request for a density map, describing what is wanted and how hard to look
+ * for it.
+ * EMD- (case-insensitively) is taken as an EMDB entry, anything
+ * else as a PDB entry.
+ *
+ * @param id a PDB or EMDB identifier
+ * @return a new builder
+ */
+ public static Builder builder(String id) {
+ if (id == null) {
+ throw new IllegalArgumentException("Identifier must not be null");
+ }
+ String trimmed = id.trim();
+ if (trimmed.toUpperCase().startsWith("EMD-") || trimmed.toUpperCase().startsWith("EMD_")) {
+ return new Builder(null, normalizeEmdbId(trimmed)).kind(DensityMapKind.EM);
+ }
+ return new Builder(new PdbId(trimmed), null);
+ }
+
+ /**
+ * Normalises an EMDB identifier to the canonical EMD-1234 form.
+ *
+ * @param emdbId an identifier such as emd-1234, EMD_1234
+ * or a bare number
+ * @return the canonical form
+ */
+ public static String normalizeEmdbId(String emdbId) {
+ if (emdbId == null) {
+ return null;
+ }
+ String digits = emdbId.trim().toUpperCase().replaceFirst("^EMD[-_]?", "");
+ return "EMD-" + digits;
+ }
+
+ /**
+ * Extracts the numeric part of an EMDB identifier, as used in file names and
+ * some URL templates.
+ *
+ * @param emdbId an EMDB identifier in any accepted form
+ * @return the digits, without the EMD- prefix
+ */
+ public static String emdbNumber(String emdbId) {
+ return normalizeEmdbId(emdbId).substring("EMD-".length());
+ }
+
+ /** @return the PDB entry requested, or null for an EMDB-only request */
+ public PdbId getPdbId() {
+ return pdbId;
+ }
+
+ /** @return the EMDB entry, if known or explicitly requested; otherwise null */
+ public String getEmdbId() {
+ return emdbId;
+ }
+
+ /** @return the kind of map wanted; never null */
+ public DensityMapKind getKind() {
+ return kind;
+ }
+
+ /** @return how aggressively to re-fetch, or null to use the cache's setting */
+ public FetchBehavior getFetchBehavior() {
+ return fetchBehavior;
+ }
+
+ /** @return an override for the cache directory, or null to use the cache's own */
+ public File getCacheDir() {
+ return cacheDir;
+ }
+
+ /**
+ * Whether sources that deliver something a viewer cannot contour directly
+ * — currently only map coefficients — may be used.
+ *
+ * @return true by default; a viewer should set it to
+ * false
+ */
+ public boolean isAllowNonRenderableFormats() {
+ return allowNonRenderableFormats;
+ }
+
+ /** @return the download size limit in bytes, or 0 for no limit */
+ public long getMaxDownloadBytes() {
+ return maxDownloadBytes;
+ }
+
+ /** @return an explicit source order, or null to let the cache choose */
+ public Listnull means {@link DensityMapKind#AUTO} */
+ public Builder kind(DensityMapKind kind) {
+ this.kind = kind == null ? DensityMapKind.AUTO : kind;
+ return this;
+ }
+
+ /** @param emdbId the EMDB entry to use, skipping the PDB-to-EMDB lookup */
+ public Builder emdbId(String emdbId) {
+ this.emdbId = emdbId == null ? null : normalizeEmdbId(emdbId);
+ return this;
+ }
+
+ /** @param fetchBehavior how aggressively to re-fetch */
+ public Builder fetchBehavior(FetchBehavior fetchBehavior) {
+ this.fetchBehavior = fetchBehavior;
+ return this;
+ }
+
+ /** @param cacheDir an override for the cache directory */
+ public Builder cacheDir(File cacheDir) {
+ this.cacheDir = cacheDir;
+ return this;
+ }
+
+ /**
+ * @param allow whether formats that cannot be contoured directly may be used.
+ * A viewer should pass false.
+ */
+ public Builder allowNonRenderableFormats(boolean allow) {
+ this.allowNonRenderableFormats = allow;
+ return this;
+ }
+
+ /** @param bytes the download size limit, or 0 for no limit, or a negative value to use the cache's setting */
+ public Builder maxDownloadBytes(long bytes) {
+ this.maxDownloadBytes = bytes;
+ return this;
+ }
+
+ /** @param chain an explicit source order, overriding the cache's choice */
+ public Builder sourceChain(List.meta sidecar beside the cached
+ * file, so a result can be reconstructed later without contacting any server.
+ * That is what lets the cache serve
+ * {@link org.biojava.nbio.structure.io.LocalPDBDirectory.FetchBehavior#LOCAL_ONLY}
+ * requests fully offline.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class DensityMapResult {
+
+ private static final Logger logger = LoggerFactory.getLogger(DensityMapResult.class);
+
+ /** Extension of the metadata sidecar written beside every cached map. */
+ public static final String META_EXT = ".meta";
+
+ private final File file;
+ private final DensityMapSource source;
+ private final DensityFileFormat format;
+ private final DensityMapKind kind;
+ private final PdbId pdbId;
+ private final String emdbId;
+ private final String sourceUrl;
+ private final boolean fromCache;
+ private final Double recommendedContourLevel;
+ private final Double sigma;
+
+ /**
+ * @param file the cached map file
+ * @param source which service supplied it
+ * @param format the file format
+ * @param kind what the values mean; never {@link DensityMapKind#AUTO}
+ * @param pdbId the PDB entry, may be null
+ * @param emdbId the EMDB entry, may be null
+ * @param sourceUrl the URL it came from
+ * @param fromCache whether it was already on disk rather than freshly downloaded
+ * @param recommendedContourLevel the author-recommended contour level in absolute
+ * map units, or null if unknown
+ * @param sigma the RMS deviation of the map, or null if unknown
+ */
+ public DensityMapResult(File file, DensityMapSource source, DensityFileFormat format, DensityMapKind kind,
+ PdbId pdbId, String emdbId, String sourceUrl, boolean fromCache,
+ Double recommendedContourLevel, Double sigma) {
+ this.file = file;
+ this.source = source;
+ this.format = format;
+ this.kind = kind;
+ this.pdbId = pdbId;
+ this.emdbId = emdbId;
+ this.sourceUrl = sourceUrl;
+ this.fromCache = fromCache;
+ this.recommendedContourLevel = recommendedContourLevel;
+ this.sigma = sigma;
+ }
+
+ /** @return the cached map file */
+ public File getFile() {
+ return file;
+ }
+
+ /** @return which service supplied the map */
+ public DensityMapSource getSource() {
+ return source;
+ }
+
+ /** @return the file format */
+ public DensityFileFormat getFormat() {
+ return format;
+ }
+
+ /** @return what the map values mean; never {@link DensityMapKind#AUTO} */
+ public DensityMapKind getKind() {
+ return kind;
+ }
+
+ /** @return the PDB entry, or null */
+ public PdbId getPdbId() {
+ return pdbId;
+ }
+
+ /** @return the EMDB entry, or null */
+ public String getEmdbId() {
+ return emdbId;
+ }
+
+ /** @return the URL the map was fetched from */
+ public String getSourceUrl() {
+ return sourceUrl;
+ }
+
+ /** @return true if the file was already cached rather than downloaded now */
+ public boolean isFromCache() {
+ return fromCache;
+ }
+
+ /**
+ * Whether a viewer can contour this file as it stands.
+ *
+ * @return false for map coefficients, which need a Fourier
+ * transform first
+ * @see DensityFileFormat#isJmolLoadable()
+ */
+ public boolean isRenderable() {
+ return format != null && format.isJmolLoadable();
+ }
+
+ /**
+ * The contour level recommended by the depositors, in absolute map units.
+ * Normally only available for EMDB maps, where it is the conventional way to
+ * contour rather than a multiple of sigma.
+ *
+ * @return the level, or null if unknown
+ */
+ public Double getRecommendedContourLevel() {
+ return recommendedContourLevel;
+ }
+
+ /**
+ * @return the RMS deviation of the map values, or null if unknown
+ */
+ public Double getSigma() {
+ return sigma;
+ }
+
+ /**
+ * The recommended contour expressed in multiples of sigma, for viewers that
+ * prefer to work that way.
+ *
+ * @return the level divided by sigma, or null if either is unknown
+ * or sigma is zero
+ */
+ public Double getContourInSigma() {
+ if (recommendedContourLevel == null || sigma == null || sigma == 0.0) {
+ return null;
+ }
+ return recommendedContourLevel / sigma;
+ }
+
+ /** @return the size of the cached file in bytes, or 0 if it is missing */
+ public long getFileSizeBytes() {
+ return file == null ? 0 : file.length();
+ }
+
+ /**
+ * The metadata sidecar file for a cached map.
+ *
+ * @param mapFile the cached map
+ * @return the sidecar path, which need not exist
+ */
+ public static File metaFileFor(File mapFile) {
+ return new File(mapFile.getAbsoluteFile().getParentFile(), mapFile.getName() + META_EXT);
+ }
+
+ /**
+ * Writes this result's metadata beside the cached file, so that it can be
+ * reconstructed later without any network access.
+ */
+ public void writeMeta() {
+ Properties p = new Properties();
+ put(p, "source", source);
+ put(p, "format", format);
+ put(p, "kind", kind);
+ put(p, "pdbId", pdbId == null ? null : pdbId.getId());
+ put(p, "emdbId", emdbId);
+ put(p, "url", sourceUrl);
+ put(p, "downloaded", Instant.now().toString());
+ put(p, "bytes", getFileSizeBytes());
+ put(p, "contourLevel", recommendedContourLevel);
+ put(p, "sigma", sigma);
+ File meta = metaFileFor(file);
+ try (OutputStream out = Files.newOutputStream(meta.toPath())) {
+ p.store(out, "BioJava density map metadata");
+ } catch (IOException e) {
+ // Losing the sidecar costs us the offline description, not the map itself.
+ logger.warn("Could not write density metadata [{}]: {}", meta, e.getMessage());
+ }
+ }
+
+ /**
+ * Reconstructs a result from a cached file and its metadata sidecar.
+ *
+ * @param mapFile the cached map
+ * @return the reconstructed result, or null if the sidecar is
+ * missing or unusable
+ */
+ public static DensityMapResult readMeta(File mapFile) {
+ File meta = metaFileFor(mapFile);
+ if (!meta.isFile()) {
+ return null;
+ }
+ Properties p = new Properties();
+ try (InputStream in = Files.newInputStream(meta.toPath())) {
+ p.load(in);
+ } catch (IOException e) {
+ logger.warn("Could not read density metadata [{}]: {}", meta, e.getMessage());
+ return null;
+ }
+ try {
+ String pdb = p.getProperty("pdbId");
+ return new DensityMapResult(mapFile,
+ DensityMapSource.valueOf(p.getProperty("source")),
+ DensityFileFormat.valueOf(p.getProperty("format")),
+ DensityMapKind.valueOf(p.getProperty("kind")),
+ pdb == null || pdb.isEmpty() ? null : new PdbId(pdb),
+ emptyToNull(p.getProperty("emdbId")),
+ p.getProperty("url"),
+ true,
+ parseDouble(p.getProperty("contourLevel")),
+ parseDouble(p.getProperty("sigma")));
+ } catch (RuntimeException e) {
+ logger.warn("Density metadata [{}] is unusable: {}", meta, e.getMessage());
+ return null;
+ }
+ }
+
+ private static void put(Properties p, String key, Object value) {
+ p.setProperty(key, value == null ? "" : String.valueOf(value));
+ }
+
+ private static String emptyToNull(String s) {
+ return s == null || s.isEmpty() ? null : s;
+ }
+
+ private static Double parseDouble(String s) {
+ if (s == null || s.isEmpty()) {
+ return null;
+ }
+ try {
+ return Double.valueOf(s);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s %s map from %s (%s, %d bytes)%s",
+ pdbId == null ? emdbId : pdbId.getId(), kind, source, format, getFileSizeBytes(),
+ isRenderable() ? "" : " [needs an FFT before it can be displayed]");
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapSource.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapSource.java
new file mode 100644
index 0000000000..a0f2e1a365
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapSource.java
@@ -0,0 +1,84 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+/**
+ * A remote service that density data can be fetched from.
+ * edmaps.rcsb.org service, which used to serve
+ * DSN6 and MTZ files, was shut down in October 2024. The map coefficients
+ * published with the wwPDB validation reports replaced it, but those are
+ * structure factors rather than a sampled grid; see
+ * {@link DensityFileFormat#MAP_COEFFICIENTS_CIF_GZ}.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public enum DensityMapSource {
+
+ /**
+ * RCSB's Mol* density server at maps.rcsb.org, which serves
+ * downsampled BinaryCIF volume slices for both X-ray and EM entries.
+ */
+ RCSB_VOLUME_SERVER("rcsbvs"),
+
+ /**
+ * PDBe's Mol* density server, equivalent to {@link #RCSB_VOLUME_SERVER}.
+ */
+ PDBE_VOLUME_SERVER("pdbevs"),
+
+ /**
+ * PDBe's pre-computed full CCP4 maps. This is the source Jmol itself uses for
+ * its built-in map-loading shortcuts.
+ */
+ PDBE_CCP4("pdbe"),
+
+ /**
+ * The primary map of an EMDB entry, at full resolution. Can be very large.
+ */
+ EMDB_MAP("emdb"),
+
+ /**
+ * Map coefficients from the wwPDB validation reports. Archival only: these
+ * cannot be displayed without an FFT.
+ */
+ WWPDB_MAP_COEFFICIENTS("wwpdb");
+
+ private final String fileToken;
+
+ DensityMapSource(String fileToken) {
+ this.fileToken = fileToken;
+ }
+
+ /**
+ * The short token used to distinguish this source in a cached file name, so
+ * that maps of the same kind from different sources never collide.
+ *
+ * @return the token
+ */
+ public String getFileToken() {
+ return fileToken;
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapTooLargeException.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapTooLargeException.java
new file mode 100644
index 0000000000..ce62e36d1d
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapTooLargeException.java
@@ -0,0 +1,84 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.io.IOException;
+
+/**
+ * Thrown when a density map exceeds the configured download size limit.
+ * null
+ * @param sigma the RMS deviation of the map values, or null
+ * @param mapSizeBytes the size of the primary map file, or null
+ */
+ public EmdbEntryInfo(String emdbId, Double recommendedContourLevel, Double sigma, Long mapSizeBytes) {
+ this.emdbId = emdbId;
+ this.recommendedContourLevel = recommendedContourLevel;
+ this.sigma = sigma;
+ this.mapSizeBytes = mapSizeBytes;
+ }
+
+ /** @return the entry identifier in canonical EMD-1234 form */
+ public String getEmdbId() {
+ return emdbId;
+ }
+
+ /**
+ * The contour level the depositors recommend, in absolute map units. EM maps
+ * are conventionally displayed at this level rather than at a multiple of
+ * sigma, so it is the right default for a viewer.
+ *
+ * @return the level, or null if the entry does not state one
+ */
+ public Double getRecommendedContourLevel() {
+ return recommendedContourLevel;
+ }
+
+ /**
+ * @return the RMS deviation of the map values, which converts between absolute
+ * and sigma-relative contour levels, or null if unknown
+ */
+ public Double getSigma() {
+ return sigma;
+ }
+
+ /**
+ * @return the size of the full primary map in bytes, or null if
+ * unknown. Used to decline a download before it starts.
+ */
+ public Long getMapSizeBytes() {
+ return mapSizeBytes;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s[contour=%s, sigma=%s, %s bytes]",
+ emdbId, recommendedContourLevel, sigma, mapSizeBytes);
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryResolver.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryResolver.java
new file mode 100644
index 0000000000..fab0101b34
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryResolver.java
@@ -0,0 +1,382 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.biojava.nbio.structure.PdbId;
+import org.biojava.nbio.structure.align.util.URLConnectionTools;
+import org.biojava.nbio.structure.io.LocalPDBDirectory.FetchBehavior;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Finds the EMDB entry, if any, associated with a PDB entry, and reads the few
+ * facts about its map that are needed to fetch and display it.
+ * null if unknown
+ */
+ public Double getContourLevelFor(PdbId pdbId) {
+ Mapping cached = readMapping(pdbId);
+ if (cached != null && (fetchBehavior == FetchBehavior.LOCAL_ONLY || cached.isFresh(mappingMaxAgeDays))) {
+ return cached.contourLevel;
+ }
+ getEmdbIds(pdbId);
+ Mapping refreshed = readMapping(pdbId);
+ return refreshed == null ? null : refreshed.contourLevel;
+ }
+
+ /**
+ * Reads an EMDB entry's map metadata: contour level, RMS deviation and the size
+ * of the primary map file.
+ * null if it could not be obtained
+ */
+ public EmdbEntryInfo getEntryInfo(String emdbId) {
+ String canonical = DensityMapRequest.normalizeEmdbId(emdbId);
+ File cacheFile = DensityCacheLayout.emdbMapInfoFile(cacheRoot, canonical);
+
+ String json = null;
+ if (cacheFile.isFile()) {
+ try {
+ json = new String(Files.readAllBytes(cacheFile.toPath()), StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ logger.debug("Could not read cached EMDB metadata [{}]: {}", cacheFile, e.getMessage());
+ }
+ }
+ if (json == null) {
+ if (fetchBehavior == FetchBehavior.LOCAL_ONLY) {
+ return null;
+ }
+ String url = UrlTemplates.expand(mapInfoUrlTemplate, UrlTemplates.values(null, canonical, -1));
+ try {
+ json = read(new URL(url));
+ } catch (IOException e) {
+ logger.warn("Could not read EMDB metadata for {}: {}", canonical, e.getMessage());
+ return null;
+ }
+ writeCache(cacheFile, json);
+ }
+
+ try {
+ JsonNode map = MAPPER.readTree(json).path("map");
+ Double contour = null;
+ JsonNode contours = map.path("contour_list").path("contour");
+ for (JsonNode c : contours) {
+ if (contour == null || c.path("primary").asBoolean(false)) {
+ contour = c.path("level").isNumber() ? c.path("level").asDouble() : contour;
+ }
+ }
+ Double sigma = map.path("statistics").path("std").isNumber()
+ ? map.path("statistics").path("std").asDouble() : null;
+ Long bytes = map.path("size_kbytes").isNumber()
+ ? map.path("size_kbytes").asLong() * 1024L : null;
+ return new EmdbEntryInfo(canonical, contour, sigma, bytes);
+ } catch (IOException | RuntimeException e) {
+ logger.warn("Could not parse EMDB metadata for {}: {}", canonical, e.getMessage());
+ return null;
+ }
+ }
+
+ private Mapping searchEmdb(PdbId pdbId) {
+ String url = UrlTemplates.expand(searchUrlTemplate,
+ UrlTemplates.values(DensityCacheLayout.shortIdOrFull(pdbId), null, -1));
+ try {
+ String csv = read(new URL(url));
+ List.map.gz can be handed to it without decompressing first.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class EmdbMapProvider extends AbstractDensityMapProvider {
+
+ /** Default base URL of the EMDB archive. */
+ public static final String DEFAULT_SERVER_URL = "https://ftp.ebi.ac.uk/pub/databases/emdb/structures/";
+
+ /** Default path template for an entry's primary map. */
+ public static final String DEFAULT_MAP_TEMPLATE = "{emdb_id}/map/emd_{emdb_num}.map.gz";
+
+ private static String serverBaseUrl = DEFAULT_SERVER_URL;
+ private static String mapTemplate = DEFAULT_MAP_TEMPLATE;
+
+ private final EmdbEntryResolver resolver;
+
+ /**
+ * @param cacheRoot the BioJava cache directory
+ * @param resolver used to find the EMDB entry for a PDB entry and to read its
+ * contour level and size
+ */
+ public EmdbMapProvider(File cacheRoot, EmdbEntryResolver resolver) {
+ super(cacheRoot);
+ this.resolver = resolver;
+ }
+
+ /** @return the base URL of the EMDB archive */
+ public static String getServerBaseUrl() {
+ return serverBaseUrl;
+ }
+
+ /** @param url the base URL; a trailing slash is added if missing */
+ public static void setServerBaseUrl(String url) {
+ serverBaseUrl = url == null ? DEFAULT_SERVER_URL : (url.endsWith("/") ? url : url + "/");
+ }
+
+ /** @param template the path template for an entry's primary map */
+ public static void setMapUrlTemplate(String template) {
+ mapTemplate = template == null ? DEFAULT_MAP_TEMPLATE : template;
+ }
+
+ /** Restores the default server and template. */
+ public static void resetToDefaults() {
+ serverBaseUrl = DEFAULT_SERVER_URL;
+ mapTemplate = DEFAULT_MAP_TEMPLATE;
+ }
+
+ @Override
+ public DensityMapSource getSource() {
+ return DensityMapSource.EMDB_MAP;
+ }
+
+ @Override
+ public DensityFileFormat getFormat() {
+ return DensityFileFormat.CCP4_GZ;
+ }
+
+ @Override
+ public boolean supports(DensityMapKind kind) {
+ return kind == DensityMapKind.EM;
+ }
+
+ /**
+ * Builds the URL of an entry's primary map without fetching it.
+ *
+ * @param emdbId the EMDB entry, in any accepted form
+ * @return the URL as a string
+ */
+ public String buildUrl(String emdbId) {
+ return serverBaseUrl + UrlTemplates.expand(mapTemplate, UrlTemplates.values(null, emdbId, -1));
+ }
+
+ @Override
+ public DensityMapResult fetch(DensityMapRequest request) throws IOException {
+ if (!supports(request.getKind())) {
+ return null;
+ }
+ String emdbId = request.getEmdbId();
+ if (emdbId == null) {
+ return null;
+ }
+
+ EmdbEntryInfo info = resolver == null ? null : resolver.getEntryInfo(emdbId);
+
+ // Decline before transferring anything: EMDB reports the map size in its
+ // metadata, so an oversized map costs one small request rather than a
+ // partial multi-hundred-megabyte download.
+ long limit = effectiveMaxBytes(request);
+ if (limit > 0 && info != null && info.getMapSizeBytes() != null && info.getMapSizeBytes() > limit) {
+ URL url = new URL(buildUrl(emdbId));
+ reportTooLarge(url, info.getMapSizeBytes(), limit, request);
+ throw new DensityMapTooLargeException(url.toString(), info.getMapSizeBytes(), limit);
+ }
+
+ URL url = new URL(buildUrl(emdbId));
+ File target = DensityCacheLayout.emdbMapFile(effectiveCacheRoot(request), emdbId,
+ getSource(), getFormat(), null);
+ return obtain(request, url, target, DensityMapKind.EM, emdbId,
+ info == null ? null : info.getRecommendedContourLevel(),
+ info == null ? null : info.getSigma());
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/NoDensityMapException.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/NoDensityMapException.java
new file mode 100644
index 0000000000..6691b584d9
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/NoDensityMapException.java
@@ -0,0 +1,99 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.biojava.nbio.structure.PdbId;
+
+/**
+ * Thrown when no enabled source could supply a density map for an entry.
+ * null for a
+ * request made by EMDB identifier
+ * @param kind the kind of map that was requested
+ * @param attempts what happened at each source that was tried, in the order tried
+ */
+ public NoDensityMapException(PdbId pdbId, DensityMapKind kind, Mapnull
+ */
+ public PdbId getPdbId() {
+ return pdbId;
+ }
+
+ /**
+ * @return the kind of map that was requested
+ */
+ public DensityMapKind getKind() {
+ return kind;
+ }
+
+ /**
+ * What happened at each source, in the order they were tried. Suitable for
+ * building a human-readable explanation.
+ *
+ * @return an unmodifiable map from source to reason
+ */
+ public Map1CBS.ccp4 and pdb_00001cbs.ccp4 both
+ * return HTTP 404. Entries deposited without structure factors, and cryo-EM
+ * entries, have no maps here at all.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class PdbeCcp4MapProvider extends AbstractDensityMapProvider {
+
+ /** Default base URL of the PDBe map service. */
+ public static final String DEFAULT_SERVER_URL = "https://www.ebi.ac.uk/pdbe/coordinates/files/";
+
+ /**
+ * An equivalent base URL serving the same files, kept in the documentation as a
+ * ready alternative should the primary one change.
+ */
+ public static final String ALTERNATIVE_SERVER_URL = "https://www.ebi.ac.uk/pdbe/entry-files/";
+
+ /** Default path template for the 2mFo-DFc map. */
+ public static final String DEFAULT_TWO_FO_FC_TEMPLATE = "{pdbid_lc}.ccp4";
+
+ /** Default path template for the mFo-DFc difference map. */
+ public static final String DEFAULT_FO_FC_TEMPLATE = "{pdbid_lc}_diff.ccp4";
+
+ private static String serverBaseUrl = DEFAULT_SERVER_URL;
+ private static String twoFoFcTemplate = DEFAULT_TWO_FO_FC_TEMPLATE;
+ private static String foFcTemplate = DEFAULT_FO_FC_TEMPLATE;
+
+ /**
+ * @param cacheRoot the BioJava cache directory
+ */
+ public PdbeCcp4MapProvider(File cacheRoot) {
+ super(cacheRoot);
+ }
+
+ /** @return the base URL of the map service */
+ public static String getServerBaseUrl() {
+ return serverBaseUrl;
+ }
+
+ /**
+ * @param url the base URL of the map service; a trailing slash is added if missing
+ */
+ public static void setServerBaseUrl(String url) {
+ serverBaseUrl = url == null ? DEFAULT_SERVER_URL : (url.endsWith("/") ? url : url + "/");
+ }
+
+ /**
+ * Overrides the path template for a map kind.
+ *
+ * @param kind {@link DensityMapKind#TWO_FO_FC} or {@link DensityMapKind#FO_FC}
+ * @param template a template understood by {@link UrlTemplates}
+ */
+ public static void setPathUrlTemplate(DensityMapKind kind, String template) {
+ if (kind == DensityMapKind.TWO_FO_FC) {
+ twoFoFcTemplate = template == null ? DEFAULT_TWO_FO_FC_TEMPLATE : template;
+ } else if (kind == DensityMapKind.FO_FC) {
+ foFcTemplate = template == null ? DEFAULT_FO_FC_TEMPLATE : template;
+ } else {
+ throw new IllegalArgumentException("PDBe CCP4 maps exist only for 2Fo-Fc and Fo-Fc, not " + kind);
+ }
+ }
+
+ /** Restores the default server and templates. */
+ public static void resetToDefaults() {
+ serverBaseUrl = DEFAULT_SERVER_URL;
+ twoFoFcTemplate = DEFAULT_TWO_FO_FC_TEMPLATE;
+ foFcTemplate = DEFAULT_FO_FC_TEMPLATE;
+ }
+
+ @Override
+ public DensityMapSource getSource() {
+ return DensityMapSource.PDBE_CCP4;
+ }
+
+ @Override
+ public DensityFileFormat getFormat() {
+ return DensityFileFormat.CCP4;
+ }
+
+ @Override
+ public boolean supports(DensityMapKind kind) {
+ return kind == DensityMapKind.TWO_FO_FC || kind == DensityMapKind.FO_FC;
+ }
+
+ /**
+ * Builds the URL for a map without fetching it.
+ *
+ * @param pdbId the entry
+ * @param kind the kind of map
+ * @return the URL as a string
+ */
+ public String buildUrl(PdbId pdbId, DensityMapKind kind) {
+ String template = kind == DensityMapKind.FO_FC ? foFcTemplate : twoFoFcTemplate;
+ return serverBaseUrl + UrlTemplates.expand(template, UrlTemplates.values(urlId(pdbId), null, -1));
+ }
+
+ @Override
+ public DensityMapResult fetch(DensityMapRequest request) throws IOException {
+ if (request.getPdbId() == null || !supports(request.getKind())) {
+ return null;
+ }
+ URL url = new URL(buildUrl(request.getPdbId(), request.getKind()));
+ File target = DensityCacheLayout.pdbMapFile(effectiveCacheRoot(request), request.getPdbId(),
+ request.getKind(), getSource(), getFormat(), null);
+ return obtain(request, url, target, request.getKind(), null, null, null);
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/UrlTemplates.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/UrlTemplates.java
new file mode 100644
index 0000000000..0518f1d4a4
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/UrlTemplates.java
@@ -0,0 +1,152 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.biojava.nbio.structure.PdbId;
+import org.biojava.nbio.structure.StructureException;
+
+/**
+ * Expands the named placeholders used in the configurable density-server URL
+ * templates.
+ *
+ *
+ * A placeholder with no supplied value is left in place rather than replaced by
+ * an empty string, so that a misconfigured template produces an obviously wrong
+ * URL instead of a subtly wrong one.
+ * {pdbid}{pdbid_lc}{pdbid_uc}{mid}cb for 1cbs{extid}pdb_00001cbs for
+ * 1cbs{emdb_id}EMD-0262{emdb_num}0262{detail}{mid} and {extid} exist for mirrors rather than for
+ * the default URLs. The services BioJava fetches from resolve an entry by name
+ * alone, but a site pointing this code at its own copy of the archive has to
+ * spell out a directory path — and so does any mirror that publishes one,
+ * such as EBI. Both the present divided layout and the per-entry layout that
+ * replaces it in July 2027 are directory paths, and between them the two
+ * placeholders express either without a code change:
+ *
+ * {mid}/{pdbid_lc}/{pdbid_lc}_validation_2fo-fc_map_coef.cif.gz
+ * entries/{mid}/{extid}/validation_reports/{extid}_validation_2fo-fc_map_coef.cif.gz
+ *
+ * null
+ * @param emdbId the EMDB identifier in any accepted form, may be null
+ * @param detail the density-server detail level, or a negative value to omit it
+ * @return a map suitable for {@link #expand(String, Map)}
+ */
+ public static Mapnull if the argument is
+ * neither a short nor an extended identifier, in which case
+ * {extid} is left unexpanded rather than guessed at
+ */
+ private static String extendedId(String pdbId) {
+ try {
+ return PdbId.toExtendedId(pdbId).toLowerCase();
+ } catch (StructureException e) {
+ return null;
+ }
+ }
+}
diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/VolumeServerProvider.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/VolumeServerProvider.java
new file mode 100644
index 0000000000..1700f3a472
--- /dev/null
+++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/VolumeServerProvider.java
@@ -0,0 +1,246 @@
+/**
+ * BioJava development code
+ *
+ * This code may be freely distributed and modified under the terms of the GNU
+ * Lesser General Public Licence. This should be distributed with the code. If
+ * you do not have a copy, see:
+ *
+ * http://www.gnu.org/copyleft/lesser.html
+ *
+ * Copyright for this code is held jointly by the individual authors. These
+ * should be listed in @author doc comments.
+ *
+ * For more information on the BioJava project and its aims, or to join the
+ * biojava-l mailing list, visit the home page at:
+ *
+ * http://www.biojava.org/
+ */
+package org.biojava.nbio.structure.io.density;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+/**
+ * Fetches downsampled volume slices from a Mol* density server.
+ * Content-Length, no ETag and no
+ * Last-Modified, so the usual server-provided validation metadata is
+ * unavailable; the size recorded for a cached slice is the byte count observed
+ * during our own download instead. Detail levels are not linear either —
+ * for a small entry the response stops growing past detail 1, while for a large
+ * EM map the step from detail 2 to 3 multiplies the size several-fold.
+ *
+ * @author Amr ALHOSSARY
+ * @since 7.3.0
+ */
+public class VolumeServerProvider extends AbstractDensityMapProvider {
+
+ /**
+ * Which density server to talk to.
+ */
+ public enum Host {
+ /** RCSB's server at maps.rcsb.org. */
+ RCSB(DensityMapSource.RCSB_VOLUME_SERVER, "https://maps.rcsb.org/", 3),
+ /** PDBe's server at www.ebi.ac.uk/pdbe/volume-server. */
+ PDBE(DensityMapSource.PDBE_VOLUME_SERVER, "https://www.ebi.ac.uk/pdbe/volume-server/", 6);
+
+ private final DensityMapSource source;
+ private final String defaultBaseUrl;
+ private final int defaultDetail;
+
+ Host(DensityMapSource source, String defaultBaseUrl, int defaultDetail) {
+ this.source = source;
+ this.defaultBaseUrl = defaultBaseUrl;
+ this.defaultDetail = defaultDetail;
+ }
+
+ /** @return the source constant this host corresponds to */
+ public DensityMapSource getSource() {
+ return source;
+ }
+
+ /** @return the default base URL */
+ public String getDefaultBaseUrl() {
+ return defaultBaseUrl;
+ }
+
+ /** @return the detail level this host's own clients use by default */
+ public int getDefaultDetail() {
+ return defaultDetail;
+ }
+ }
+
+ /** Default path template for an X-ray entry's whole unit cell. */
+ public static final String DEFAULT_XRAY_CELL_TEMPLATE = "x-ray/{pdbid_lc}/cell?detail={detail}&encoding={encoding}";
+
+ /** Default path template for an EM entry's whole cell. */
+ public static final String DEFAULT_EM_CELL_TEMPLATE = "em/emd-{emdb_num}/cell?detail={detail}&encoding={encoding}";
+
+ private final Host host;
+ private String baseUrl;
+ private int detail;
+ private String encoding = "bcif";
+
+ /**
+ * @param cacheRoot the BioJava cache directory
+ * @param host which density server to use
+ */
+ public VolumeServerProvider(File cacheRoot, Host host) {
+ super(cacheRoot);
+ this.host = host;
+ this.baseUrl = host.getDefaultBaseUrl();
+ this.detail = host.getDefaultDetail();
+ }
+
+ /** @return which density server this instance uses */
+ public Host getHost() {
+ return host;
+ }
+
+ /** @return the base URL in use */
+ public String getBaseUrl() {
+ return baseUrl;
+ }
+
+ /** @param baseUrl the base URL; a trailing slash is added if missing */
+ public void setBaseUrl(String baseUrl) {
+ this.baseUrl = baseUrl == null ? host.getDefaultBaseUrl()
+ : (baseUrl.endsWith("/") ? baseUrl : baseUrl + "/");
+ }
+
+ /** @return the detail level requested from the server */
+ public int getDetail() {
+ return detail;
+ }
+
+ /**
+ * Sets the detail level. Higher means a finer grid and a larger download; the
+ * relationship is neither linear nor the same for every entry.
+ *
+ * @param detail the level, normally 0 to 6
+ */
+ public void setDetail(int detail) {
+ this.detail = detail;
+ }
+
+ /** @return the encoding requested, either bcif or cif */
+ public String getEncoding() {
+ return encoding;
+ }
+
+ /**
+ * @param encoding bcif for BinaryCIF (default, and much smaller) or
+ * cif for text
+ */
+ public void setEncoding(String encoding) {
+ this.encoding = encoding == null ? "bcif" : encoding;
+ }
+
+ @Override
+ public DensityMapSource getSource() {
+ return host.getSource();
+ }
+
+ @Override
+ public DensityFileFormat getFormat() {
+ return "cif".equalsIgnoreCase(encoding) ? DensityFileFormat.CIF_VOLUME : DensityFileFormat.BCIF_VOLUME;
+ }
+
+ @Override
+ public boolean supports(DensityMapKind kind) {
+ return kind == DensityMapKind.TWO_FO_FC || kind == DensityMapKind.FO_FC || kind == DensityMapKind.EM;
+ }
+
+ /**
+ * Builds the URL for a request without fetching it.
+ *
+ * @param request the request; its kind must be concrete
+ * @return the URL as a string, or null if the request cannot be served
+ */
+ public String buildUrl(DensityMapRequest request) {
+ if (request.getKind() == DensityMapKind.EM) {
+ if (request.getEmdbId() == null) {
+ return null;
+ }
+ return baseUrl + UrlTemplates.expand(withEncoding(DEFAULT_EM_CELL_TEMPLATE),
+ UrlTemplates.values(null, request.getEmdbId(), detail));
+ }
+ if (request.getPdbId() == null) {
+ return null;
+ }
+ return baseUrl + UrlTemplates.expand(withEncoding(DEFAULT_XRAY_CELL_TEMPLATE),
+ UrlTemplates.values(urlId(request.getPdbId()), null, detail));
+ }
+
+ private String withEncoding(String template) {
+ return template.replace("{encoding}", encoding);
+ }
+
+ @Override
+ public DensityMapResult fetch(DensityMapRequest request) throws IOException {
+ if (!supports(request.getKind())) {
+ return null;
+ }
+ String urlString = buildUrl(request);
+ if (urlString == null) {
+ return null;
+ }
+ URL url = new URL(urlString);
+
+ // The detail level changes the content, so it has to be part of the cache key.
+ String qualifier = "d" + detail;
+ File target;
+ if (request.getKind() == DensityMapKind.EM) {
+ target = DensityCacheLayout.emdbMapFile(effectiveCacheRoot(request), request.getEmdbId(),
+ getSource(), getFormat(), qualifier);
+ } else {
+ // One response carries both the 2Fo-Fc and the Fo-Fc blocks, so the two
+ // kinds share a cache entry: asking for each separately would otherwise
+ // download and store the identical file twice. Which block to read is
+ // decided at display time, not here.
+ target = DensityCacheLayout.pdbMapFile(effectiveCacheRoot(request), request.getPdbId(),
+ DensityCacheLayout.BOTH_KINDS_TOKEN, getSource(), getFormat(), qualifier);
+ }
+ DensityMapResult result = obtain(request, url, target, request.getKind(), request.getEmdbId(), null, null);
+
+ if (request.getKind() == DensityMapKind.FO_FC) {
+ return presentAsDifferenceMap(result);
+ }
+ return result;
+ }
+
+ /**
+ * Points the result at the companion file name that makes Jmol read the
+ * difference-map block. See
+ * {@link DensityCacheLayout#differenceMarkerFile(File)} for why the marker has
+ * to be in the name.
+ * gemmi sf2map, or cif2mtz followed by
+ * CCP4's fft. Nothing in BioJava or Jmol will render them.
+ * edmaps.rcsb.org was shut down in October 2024, and because they
+ * are the authoritative archival form. For anything that needs to be displayed,
+ * prefer {@link PdbeCcp4MapProvider} or {@link VolumeServerProvider}. Accordingly
+ * this source is disabled by default in {@link DensityMapCache}.
+ * ETag, so downloads from here are checksum-verified automatically.
+ *
+ * setServerBaseUrl(EBI_MIRROR_URL);
+ * setPathUrlTemplate(DensityMapKind.TWO_FO_FC, DIVIDED_TWO_FO_FC_TEMPLATE);
+ * setPathUrlTemplate(DensityMapKind.FO_FC, DIVIDED_FO_FC_TEMPLATE);
+ *
+ * Setting the base alone yields 404s, because the flat file names do not exist
+ * there.
+ */
+ public static final String EBI_MIRROR_URL = "https://ftp.ebi.ac.uk/pub/databases/pdb/validation_reports/";
+
+ /**
+ * Default path template for the 2mFo-DFc coefficients: the file name alone.
+ * .../pdb/data/.
+ *