diff --git a/biojava-integrationtest/src/test/java/org/biojava/nbio/structure/test/io/density/DensityMapIntegrationTest.java b/biojava-integrationtest/src/test/java/org/biojava/nbio/structure/test/io/density/DensityMapIntegrationTest.java new file mode 100644 index 0000000000..97bf76a294 --- /dev/null +++ b/biojava-integrationtest/src/test/java/org/biojava/nbio/structure/test/io/density/DensityMapIntegrationTest.java @@ -0,0 +1,215 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.test.io.density; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.List; + +import org.biojava.nbio.core.util.FileDownloadUtils; +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.io.density.Ccp4Header; +import org.biojava.nbio.structure.io.density.DensityFileFormat; +import org.biojava.nbio.structure.io.density.DensityMapCache; +import org.biojava.nbio.structure.io.density.DensityMapKind; +import org.biojava.nbio.structure.io.density.DensityMapRequest; +import org.biojava.nbio.structure.io.density.DensityMapResult; +import org.biojava.nbio.structure.io.density.DensityMapSource; +import org.biojava.nbio.structure.io.density.NoDensityMapException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Density fetching against the real services. + *

+ * Deliberately frugal: the entries chosen keep a full run to a couple of + * megabytes plus a few small metadata calls. In particular the cryo-EM path is + * exercised with the size limit set so low that the 116 MB map is declined + * before any of its body is transferred, which tests the whole resolution and + * guard sequence without the download. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class DensityMapIntegrationTest { + + private File cacheRoot; + private DensityMapCache cache; + + @BeforeEach + public void setUp() throws IOException { + cacheRoot = Files.createTempDirectory("bj-density-it").toFile(); + cache = new DensityMapCache(cacheRoot.getAbsolutePath()); + } + + @AfterEach + public void tearDown() throws IOException { + FileDownloadUtils.deleteDirectory(cacheRoot.toPath()); + } + + /** The default path for an X-ray entry: the smallest source answers first. */ + @Test + public void fetchesAnXrayMapFromTheFirstSourceTried() throws IOException { + DensityMapResult result = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + + assertEquals(DensityMapSource.RCSB_VOLUME_SERVER, result.getSource()); + assertEquals(DensityMapKind.TWO_FO_FC, result.getKind()); + assertTrue(result.isRenderable()); + assertFalse(result.isFromCache()); + assertTrue(result.getFileSizeBytes() > 1024); + assertTrue(DensityMapResult.metaFileFor(result.getFile()).isFile(), + "a .meta sidecar makes the result reconstructible offline"); + + // second call must come from the cache without another download + DensityMapResult again = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + assertTrue(again.isFromCache()); + assertEquals(result.getFile(), again.getFile()); + } + + /** + * Both map kinds come out of one download, and the difference map is presented + * under the companion name that makes Jmol read the other data block. + */ + @Test + public void bothKindsShareASingleDownload() throws IOException { + DensityMapResult twoFoFc = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + DensityMapResult foFc = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.FO_FC); + + assertEquals(DensityMapKind.FO_FC, foFc.getKind()); + assertFalse(twoFoFc.getFile().equals(foFc.getFile()), "the difference map needs its own file name"); + assertTrue(foFc.getFile().getName().contains("&diff=1"), + "the marker has to be in the name for Jmol to select the FO-FC block"); + assertEquals(twoFoFc.getFileSizeBytes(), foFc.getFileSizeBytes(), + "both names must address the same bytes"); + } + + /** PDBe serves real CCP4 files, which the header check should recognise. */ + @Test + public void pdbeServesAGenuineCcp4Map() throws IOException { + cache.setSourceChain(DensityMapKind.TWO_FO_FC, Arrays.asList(DensityMapSource.PDBE_CCP4)); + DensityMapResult result = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + + assertEquals(DensityMapSource.PDBE_CCP4, result.getSource()); + assertEquals(DensityFileFormat.CCP4, result.getFormat()); + assertTrue(Ccp4Header.isCcp4(result.getFile()), "the CCP4 stamp should be present at byte 208"); + assertTrue(FileDownloadUtils.validateFile(result.getFile())); + } + + /** + * The whole cryo-EM route: resolve the EMDB entry, pick up the author contour + * level, and decline the full map on size without transferring it. + */ + @Test + public void resolvesCryoEmEntriesAndHonoursTheSizeLimit() throws IOException { + List emdbIds = cache.getEmdbResolver().getEmdbIds(new PdbId("6hu9")); + assertEquals(Arrays.asList("EMD-0262"), emdbIds); + + DensityMapResult result = cache.getDensityMap(new PdbId("6hu9"), DensityMapKind.AUTO); + assertEquals(DensityMapKind.EM, result.getKind()); + assertEquals("EMD-0262", result.getEmdbId()); + assertNotNull(result.getRecommendedContourLevel(), + "EM maps need the author contour level to be displayed properly"); + assertEquals(0.0263, result.getRecommendedContourLevel(), 1e-6); + assertNotNull(result.getContourInSigma()); + + // With only the full archive enabled and a tiny ceiling, the guard must fire + // rather than pulling down 116 MB. + DensityMapCache strict = new DensityMapCache(cacheRoot.getAbsolutePath()); + strict.setSourceChain(DensityMapKind.EM, Arrays.asList(DensityMapSource.EMDB_MAP)); + strict.setMaxDownloadBytes(1024); + try { + strict.getDensityMap(DensityMapRequest.builder(new PdbId("6hu9")).kind(DensityMapKind.EM).build()); + fail("the size guard should have declined the full EMDB map"); + } catch (NoDensityMapException e) { + assertTrue(e.getAttempts().get(DensityMapSource.EMDB_MAP).contains("too large")); + } + } + + /** 4HHB was deposited in 1984 without structure factors, so nothing has a map for it. */ + @Test + public void reportsWhyAnEntryHasNoDensity() throws IOException { + cache.setSourceEnabled(DensityMapSource.WWPDB_MAP_COEFFICIENTS, true); + try { + cache.getDensityMap(new PdbId("4hhb"), DensityMapKind.AUTO); + fail("4hhb has no deposited structure factors"); + } catch (NoDensityMapException e) { + assertFalse(e.getAttempts().isEmpty()); + assertTrue(e.getAttempts().values().stream().anyMatch(r -> r.contains("404"))); + } + } + + /** + * Coefficients must arrive intact and be verifiable afterwards. Whether that + * verification includes a cryptographic digest depends on the server. + *

+ * The divided archive paths on files.wwpdb.org and files.rcsb.org return the content + * MD5 as the ETag. The flat /validation/download/ endpoint this provider now uses + * returns neither an ETag nor a Content-Length, and neither does the beta archive on + * those two hosts, so no digest can be recorded there. The size sidecar is written + * from the bytes actually read, so it exists either way. + *

+ * The digest is therefore asserted when the server offered one and skipped when it + * did not, rather than being required: requiring it would fail against the endpoint + * we use, and hard-coding the divided path would only work until the archive + * transition in July 2027. + */ + @Test + public void mapCoefficientsArriveIntactAndVerifiable() throws IOException { + cache.setSourceEnabled(DensityMapSource.WWPDB_MAP_COEFFICIENTS, true); + cache.setSourceChain(DensityMapKind.TWO_FO_FC, Arrays.asList(DensityMapSource.WWPDB_MAP_COEFFICIENTS)); + + DensityMapResult result = cache.getDensityMap(DensityMapRequest.builder(new PdbId("1cbs")) + .kind(DensityMapKind.TWO_FO_FC) + .allowNonRenderableFormats(true) + .build()); + + assertEquals(DensityMapSource.WWPDB_MAP_COEFFICIENTS, result.getSource()); + assertFalse(result.isRenderable(), + "structure factors are not a map and must not claim to be renderable"); + + // written from the observed byte count, so it is present whether or not the + // server declared a length + assertTrue(FileDownloadUtils.validateFile(result.getFile()), + "a freshly downloaded file must validate against its own sidecars"); + + File hashFile = new File(result.getFile().getParentFile(), result.getFile().getName() + ".hash_MD5"); + if (hashFile.isFile()) { + String recorded = new String(Files.readAllBytes(hashFile.toPath()), StandardCharsets.UTF_8).trim(); + assertTrue(FileDownloadUtils.verifyHash(result.getFile(), FileDownloadUtils.Hash.MD5, recorded), + "the recorded MD5 must match the file it describes"); + } else { + System.out.println("No MD5 recorded for " + result.getSourceUrl() + + " - the server offered no usable ETag. Size validation still applies."); + } + + // corrupt it and confirm validation actually catches it + Files.write(result.getFile().toPath(), new byte[] {0, 1, 2, 3}); + assertFalse(FileDownloadUtils.validateFile(result.getFile()), + "a truncated file must not validate"); + } +} diff --git a/biojava-structure-gui/src/main/java/demo/DemoShowElectronDensity.java b/biojava-structure-gui/src/main/java/demo/DemoShowElectronDensity.java new file mode 100644 index 0000000000..3948605744 --- /dev/null +++ b/biojava-structure-gui/src/main/java/demo/DemoShowElectronDensity.java @@ -0,0 +1,80 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package demo; + +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.Structure; +import org.biojava.nbio.structure.StructureIO; +import org.biojava.nbio.structure.align.gui.jmol.StructureAlignmentJmol; +import org.biojava.nbio.structure.io.density.DensityMapCache; +import org.biojava.nbio.structure.io.density.DensityMapKind; +import org.biojava.nbio.structure.io.density.DensityMapResult; + +/** + * Shows 1CBS with its electron density drawn around the bound retinoic acid. + *

+ * Both maps are displayed: the 2mFo-DFc map in blue at 1 sigma, which should hug + * the ligand closely, and the mFo-DFc difference map as a red and green pair at + * 3 sigma, which for a well refined structure should show very little. + *

+ * Once the window is up, the map can be manipulated from the Rasmol command box + * at the bottom, for instance + *

+ * isosurface ID "bj_density_2fofc" delete
+ * 
+ * to remove just the blue surface. Pressing Reset Display keeps the maps, since + * they are folded into the saved state when they are drawn. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class DemoShowElectronDensity { + + /** + * @param args an optional PDB ID to display instead of the default + * @throws Exception if the structure or the map could not be fetched + */ + public static void main(String[] args) throws Exception { + String id = args.length > 0 ? args[0] : "1cbs"; + + Structure structure = StructureIO.getStructure(id); + StructureAlignmentJmol viewer = new StructureAlignmentJmol(); + viewer.setStructure(structure); + viewer.evalString("select all; cartoon on; color chain; " + + "select ligand; wireframe 0.16; spacefill 0.4; color cpk;"); + + DensityMapCache cache = new DensityMapCache(); + System.out.println("Density cache: " + cache.getCachePath()); + + // Clipping to the ligand keeps the surface readable and, for a large map, + // keeps the contouring quick enough not to stall the interface. + for (DensityMapKind kind : new DensityMapKind[] {DensityMapKind.TWO_FO_FC, DensityMapKind.FO_FC}) { + cache.findDensityMap(new PdbId(id), kind).ifPresent(map -> { + System.out.printf("%-8s from %-20s %s (%,d bytes)%n", + map.getKind(), map.getSource(), map.getFile().getName(), map.getFileSizeBytes()); + viewer.getJmolPanel().loadDensityMap(map, "{ligand}", 5.0); + }); + } + + DensityMapResult any = cache.findDensityMap(new PdbId(id), DensityMapKind.AUTO).orElse(null); + if (any == null) { + System.out.println("No density is available for " + id + + " - try an entry with deposited structure factors, such as 1cbs."); + } + } +} diff --git a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MenuCreator.java b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MenuCreator.java index 6a0e3c6e1a..197802a2c5 100644 --- a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MenuCreator.java +++ b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MenuCreator.java @@ -66,6 +66,8 @@ public class MenuCreator { public static final String PAIRWISE_ALIGN = "New Pairwise Alignment"; public static final String MULTIPLE_ALIGN = "New Multiple Alignment"; public static final String PHYLOGENETIC_TREE = "Phylogenetic Tree"; + /** @since 7.3.0 */ + public static final String SHOW_DENSITY = "Show Electron Density"; protected static final int keyMask = Toolkit.getDefaultToolkit().getMenuShortcutKeyMask(); @@ -169,6 +171,8 @@ public static JMenuBar initJmolMenu(JFrame frame, distMax.setMnemonic(KeyEvent.VK_D); distMax.addActionListener(new MyDistMaxListener(parent)); view.add(distMax); + //Electron density + view.add(getShowDensityMenuItem(parent)); //Dot Plot - only if the alignment was an afpChain if (afpChain != null){ JMenuItem dotplot = new JMenuItem(DOT_PLOT); @@ -229,6 +233,22 @@ public static JMenuItem getOpenPDBMenuItem() { return openI; } + /** + * Menu item that fetches and displays the electron density or cryo-EM map for + * the structure currently on screen. + * + * @param parent the viewer to draw the map into + * @return the menu item + * @author Amr ALHOSSARY + * @since 7.3.0 + */ + public static JMenuItem getShowDensityMenuItem(AbstractAlignmentJmol parent) { + JMenuItem densityI = new JMenuItem(SHOW_DENSITY); + densityI.setMnemonic(KeyEvent.VK_E); + densityI.addActionListener(new MyShowDensityListener(parent)); + return densityI; + } + public static JMenuItem getLoadMenuItem() { diff --git a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MyShowDensityListener.java b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MyShowDensityListener.java new file mode 100644 index 0000000000..b1a9cf23ff --- /dev/null +++ b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/MyShowDensityListener.java @@ -0,0 +1,202 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.align.gui; + +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import javax.swing.JOptionPane; +import javax.swing.SwingWorker; + +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.Structure; +import org.biojava.nbio.structure.align.gui.jmol.AbstractAlignmentJmol; +import org.biojava.nbio.structure.io.density.DensityMapCache; +import org.biojava.nbio.structure.io.density.DensityMapKind; +import org.biojava.nbio.structure.io.density.DensityMapRequest; +import org.biojava.nbio.structure.io.density.DensityMapResult; +import org.biojava.nbio.structure.io.density.DensityMapSource; +import org.biojava.nbio.structure.io.density.NoDensityMapException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Fetches the density map for the structure on screen and draws it. + *

+ * The download happens on a {@link SwingWorker} rather than the event dispatch + * thread. Even the smallest source runs to a few hundred kilobytes and a + * full-resolution map can be far larger, so fetching inline would freeze the + * interface for as long as it took. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class MyShowDensityListener implements ActionListener { + + private static final Logger logger = LoggerFactory.getLogger(MyShowDensityListener.class); + + private static final String OPTION_2FOFC = "2Fo-Fc (electron density)"; + private static final String OPTION_FOFC = "Fo-Fc (difference)"; + private static final String OPTION_BOTH = "Both"; + private static final String OPTION_AUTO = "Whatever is available"; + + private final AbstractAlignmentJmol parent; + + /** + * @param parent the viewer to draw into + */ + public MyShowDensityListener(AbstractAlignmentJmol parent) { + this.parent = parent; + } + + @Override + public void actionPerformed(ActionEvent e) { + Structure structure = parent == null ? null : parent.getStructure(); + if (structure == null) { + JOptionPane.showMessageDialog(parent == null ? null : parent.getFrame(), + "There is no structure on screen to fetch a density map for.", + "Show Electron Density", JOptionPane.INFORMATION_MESSAGE); + return; + } + + PdbId pdbId = structure.getPdbId(); + if (pdbId == null) { + String typed = JOptionPane.showInputDialog(parent.getFrame(), + "This structure has no PDB ID. Enter one to look up its density map:", + "Show Electron Density", JOptionPane.QUESTION_MESSAGE); + if (typed == null || typed.trim().isEmpty()) { + return; + } + try { + pdbId = new PdbId(typed.trim()); + } catch (IllegalArgumentException ex) { + JOptionPane.showMessageDialog(parent.getFrame(), typed + " is not a valid PDB ID.", + "Show Electron Density", JOptionPane.ERROR_MESSAGE); + return; + } + } + + Object[] options = {OPTION_2FOFC, OPTION_FOFC, OPTION_BOTH, OPTION_AUTO}; + Object choice = JOptionPane.showInputDialog(parent.getFrame(), + "Which map would you like to see for " + pdbId.getId() + "?", + "Show Electron Density", JOptionPane.QUESTION_MESSAGE, null, options, OPTION_2FOFC); + if (choice == null) { + return; + } + + List kinds = new ArrayList<>(2); + if (OPTION_BOTH.equals(choice)) { + kinds.add(DensityMapKind.TWO_FO_FC); + kinds.add(DensityMapKind.FO_FC); + } else if (OPTION_FOFC.equals(choice)) { + kinds.add(DensityMapKind.FO_FC); + } else if (OPTION_AUTO.equals(choice)) { + kinds.add(DensityMapKind.AUTO); + } else { + kinds.add(DensityMapKind.TWO_FO_FC); + } + + fetchAndShow(pdbId, kinds); + } + + private void fetchAndShow(PdbId pdbId, List kinds) { + parent.setStatus("Fetching density map for " + pdbId.getId() + " ..."); + + new SwingWorker, Void>() { + + private NoDensityMapException missing; + + @Override + protected List doInBackground() throws Exception { + DensityMapCache cache = DensityMapCache.getInstance(); + List results = new ArrayList<>(kinds.size()); + for (DensityMapKind kind : kinds) { + try { + // Restricting to displayable formats also excludes the map + // coefficient source automatically. + results.add(cache.getDensityMap(DensityMapRequest.builder(pdbId) + .kind(kind) + .allowNonRenderableFormats(false) + .build())); + } catch (NoDensityMapException ex) { + missing = ex; + } + } + return results; + } + + @Override + protected void done() { + List results; + try { + results = get(); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return; + } catch (ExecutionException ex) { + logger.error("Could not fetch a density map for {}", pdbId.getId(), ex.getCause()); + parent.setStatus("Could not fetch density map"); + JOptionPane.showMessageDialog(parent.getFrame(), + "Could not fetch the density map:\n" + ex.getCause().getMessage(), + "Show Electron Density", JOptionPane.ERROR_MESSAGE); + return; + } + + for (DensityMapResult result : results) { + parent.getJmolPanel().loadDensityMap(result); + } + + if (results.isEmpty()) { + parent.setStatus("No density map available"); + JOptionPane.showMessageDialog(parent.getFrame(), explain(pdbId, missing), + "Show Electron Density", JOptionPane.INFORMATION_MESSAGE); + } else { + DensityMapResult first = results.get(0); + parent.setStatus(String.format("Density from %s (%,d kB)", + first.getSource(), first.getFileSizeBytes() / 1024)); + } + } + }.execute(); + } + + /** + * Turns the per-source reasons into something a user can act on, rather than a + * list of HTTP codes. + */ + private static String explain(PdbId pdbId, NoDensityMapException missing) { + StringBuilder sb = new StringBuilder("No density map is available for ") + .append(pdbId.getId()).append(".\n\n"); + if (missing == null) { + return sb.toString(); + } + Map attempts = missing.getAttempts(); + boolean allNotFound = !attempts.isEmpty() && attempts.values().stream() + .allMatch(reason -> reason.startsWith("HTTP 404") || reason.startsWith("no ")); + if (allNotFound) { + sb.append("The most likely reason is that no structure factors were deposited\n") + .append("for this entry, and that it has no associated EMDB map.\n\n"); + } + sb.append("Sources tried:\n"); + attempts.forEach((source, reason) -> sb.append(" ").append(source).append(": ").append(reason).append('\n')); + return sb.toString(); + } +} diff --git a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/AbstractAlignmentJmol.java b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/AbstractAlignmentJmol.java index 535ad182fe..86be61d8fc 100644 --- a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/AbstractAlignmentJmol.java +++ b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/AbstractAlignmentJmol.java @@ -194,6 +194,29 @@ public Structure getStructure(){ */ public abstract List getDistanceMatrices(); + /** + * The window this viewer lives in, for use as the owner of dialogs raised from + * outside this package. + * + * @return the frame, which may be null before the window is built + * @since 7.3.0 + */ + public JFrame getFrame() { + return frame; + } + + /** + * Writes a short message into the viewer's status field. + * + * @param message the message; ignored if there is no status field yet + * @since 7.3.0 + */ + public void setStatus(String message) { + if (status != null) { + status.setText(message); + } + } + /** * Set the title of the AlignmentJmol window. * @param title diff --git a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/JmolPanel.java b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/JmolPanel.java index 23361e62ed..9108489d29 100644 --- a/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/JmolPanel.java +++ b/biojava-structure-gui/src/main/java/org/biojava/nbio/structure/align/gui/jmol/JmolPanel.java @@ -31,12 +31,14 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.io.BufferedInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.PipedInputStream; import java.io.PipedOutputStream; import java.text.DecimalFormat; import java.util.List; +import java.util.Locale; import javax.swing.JComboBox; @@ -50,6 +52,8 @@ import org.biojava.nbio.structure.domain.pdp.Domain; import org.biojava.nbio.structure.domain.pdp.Segment; import org.biojava.nbio.structure.gui.util.color.ColorUtils; +import org.biojava.nbio.structure.io.density.DensityMapKind; +import org.biojava.nbio.structure.io.density.DensityMapResult; import org.biojava.nbio.structure.io.mmtf.MmtfActions; import org.biojava.nbio.structure.jama.Matrix; import org.biojava.nbio.structure.scop.ScopDatabase; @@ -166,6 +170,157 @@ public void setStructure(final Structure s) { setStructure(s, false); } + /** Isosurface id used for the 2mFo-DFc map, so that it can be addressed on its own. */ + public static final String ISOSURFACE_ID_2FOFC = "bj_density_2fofc"; + + /** Isosurface id used for the mFo-DFc difference map, drawn as a signed pair of lobes. */ + public static final String ISOSURFACE_ID_FOFC = "bj_density_fofc"; + + /** Isosurface id used for a cryo-EM map. */ + public static final String ISOSURFACE_ID_EM = "bj_density_em"; + + /** Default clipping radius, in Angstroms, around the selected atoms. */ + public static final double DEFAULT_WITHIN_RADIUS = 5.0; + + /** + * Displays a density map fetched through + * {@link org.biojava.nbio.structure.io.density.DensityMapCache}, clipped to + * {@value #DEFAULT_WITHIN_RADIUS} Angstroms around the whole model. + * + * @param map the map to display + * @throws IllegalArgumentException if the map cannot be displayed without a + * Fourier transform first + * @since 7.3.0 + */ + public void loadDensityMap(DensityMapResult map) { + loadDensityMap(map, "{*}", DEFAULT_WITHIN_RADIUS); + } + + /** + * Displays a density map, clipped to a distance around a selection. + *

+ * Contouring follows the convention for each kind of map: the 2mFo-DFc map at + * 1 sigma in blue, the mFo-DFc difference map as a signed red/green pair at 3 + * sigma, and a cryo-EM map at the level its depositors recommend when the entry + * states one, falling back to 3 sigma when it does not. + *

+ * Clipping matters for more than tidiness: contouring a whole cryo-EM grid at + * mesh resolution can take long enough to make the interface appear frozen. + * + * @param map the map to display + * @param atomSelection a Jmol atom expression such as {*} or + * {ligand}, or null to contour the whole cell + * @param withinRadius the clipping radius in Angstroms, ignored when + * atomSelection is null + * @throws IllegalArgumentException if the map cannot be displayed without a + * Fourier transform first + * @since 7.3.0 + */ + public void loadDensityMap(DensityMapResult map, String atomSelection, double withinRadius) { + if (!map.isRenderable()) { + throw new IllegalArgumentException("A " + map.getFormat() + " file holds structure factors, not a " + + "sampled map, and cannot be displayed as it stands. Convert it first, for example with " + + "'gemmi sf2map', or fetch the map from a source that serves a grid."); + } + + Double level = map.getRecommendedContourLevel(); + if (map.getKind() == DensityMapKind.EM && level != null) { + // EM maps are conventionally contoured at the absolute level the + // depositors chose rather than at a multiple of sigma. + loadDensityMap(map.getFile(), map.getKind(), level, false, atomSelection, withinRadius); + } else { + double sigma = map.getKind() == DensityMapKind.TWO_FO_FC ? 1.0 : 3.0; + loadDensityMap(map.getFile(), map.getKind(), sigma, true, atomSelection, withinRadius); + } + } + + /** + * Displays a density map file directly. + * + * @param mapFile the map file, in a format Jmol can contour (CCP4/MRC, or a + * BinaryCIF volume) + * @param kind what the map values mean, which decides the colouring and whether + * a signed pair of surfaces is drawn + * @param level the contour level + * @param levelIsSigma whether level is a multiple of the map's RMS + * deviation rather than an absolute value + * @param atomSelection a Jmol atom expression, or null to contour + * the whole cell + * @param withinRadius the clipping radius in Angstroms + * @since 7.3.0 + */ + public void loadDensityMap(File mapFile, DensityMapKind kind, double level, boolean levelIsSigma, + String atomSelection, double withinRadius) { + + String url = toJmolFileUrl(mapFile); + String within = atomSelection == null ? "" + : String.format(Locale.US, " within %.1f %s", withinRadius, atomSelection); + + if (kind == DensityMapKind.FO_FC) { + // One signed surface carrying both lobes, red for negative and green for + // positive. Drawing the negative lobe as a separate surface at "sigma -3" + // does not work: Jmol gives a negative sigma its own internal meaning and + // silently contours at the default level instead. + evalString(isosurfaceCommand(ISOSURFACE_ID_FOFC, "sign red green", level, levelIsSigma, within, url)); + } else if (kind == DensityMapKind.EM) { + evalString(isosurfaceCommand(ISOSURFACE_ID_EM, "color grey", level, levelIsSigma, within, url)); + } else { + evalString(isosurfaceCommand(ISOSURFACE_ID_2FOFC, "color blue", level, levelIsSigma, within, url)); + } + + // resetDisplay() restores "state_1", which is saved when the structure is + // loaded. Without re-saving here, pressing Reset Display would silently + // discard the map the user just asked for. + evalString("save STATE state_1"); + } + + /** + * Builds an isosurface command. + *

+ * The option order is not a matter of taste: mesh and + * nofill have to follow the file name. Placed before it, Jmol + * accepts the command without complaint and draws nothing at all. + */ + private static String isosurfaceCommand(String id, String colouring, double level, boolean levelIsSigma, + String within, String url) { + return String.format(Locale.US, + "isosurface ID \"%s\" delete; isosurface ID \"%s\" %s %s %.4f%s \"%s\" mesh nofill;", + id, id, colouring, levelIsSigma ? "sigma" : "cutoff", level, within, url); + } + + /** + * Removes any density surfaces this panel has drawn, leaving other isosurfaces + * alone. + * + * @since 7.3.0 + */ + public void clearDensityMaps() { + for (String id : new String[] {ISOSURFACE_ID_2FOFC, ISOSURFACE_ID_FOFC, ISOSURFACE_ID_EM}) { + evalString("isosurface ID \"" + id + "\" delete;"); + } + evalString("save STATE state_1"); + } + + /** + * Converts a file to the URL form Jmol expects. + *

+ * Going through {@link File#toURI()} percent-encodes spaces and removes + * backslashes, which Jmol would otherwise read as escape characters in a script + * string. On Windows the result is a single-slash file:/C:/..., + * which is normalised here to the usual three-slash form. + * + * @param file the file + * @return a URL string safe to embed in a Jmol script + * @since 7.3.0 + */ + public static String toJmolFileUrl(File file) { + String url = file.getAbsoluteFile().toURI().toString(); + if (url.startsWith("file:/") && !url.startsWith("file://")) { + url = "file:///" + url.substring("file:/".length()); + } + return url; + } + /** assign a custom color to the Jmol chains command. * */ diff --git a/biojava-structure/src/main/java/demo/DemoFetchElectronDensity.java b/biojava-structure/src/main/java/demo/DemoFetchElectronDensity.java new file mode 100644 index 0000000000..f987fd7cfe --- /dev/null +++ b/biojava-structure/src/main/java/demo/DemoFetchElectronDensity.java @@ -0,0 +1,90 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package demo; + +import java.io.IOException; + +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.io.density.DensityMapCache; +import org.biojava.nbio.structure.io.density.DensityMapKind; +import org.biojava.nbio.structure.io.density.DensityMapResult; +import org.biojava.nbio.structure.io.density.NoDensityMapException; + +/** + * Fetches electron density and cryo-EM maps, printing which source answered. + *

+ * The three entries chosen exercise the three outcomes the fallback chain has to + * handle: + *

+ * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class DemoFetchElectronDensity { + + /** + * @param args ignored + * @throws IOException if a server could not be reached at all + */ + public static void main(String[] args) throws IOException { + DensityMapCache cache = new DensityMapCache(); + System.out.println("Caching under: " + cache.getCachePath()); + System.out.println(); + + show(cache, "1cbs", DensityMapKind.TWO_FO_FC); + show(cache, "1cbs", DensityMapKind.FO_FC); + show(cache, "6hu9", DensityMapKind.AUTO); + show(cache, "4hhb", DensityMapKind.AUTO); + } + + private static void show(DensityMapCache cache, String id, DensityMapKind kind) throws IOException { + System.out.printf("%s (%s)%n", id, kind); + try { + DensityMapResult result = cache.getDensityMap(new PdbId(id), kind); + System.out.printf(" source : %s%n", result.getSource()); + System.out.printf(" format : %s%s%n", result.getFormat(), + result.isRenderable() ? "" : " (needs an FFT before display)"); + System.out.printf(" kind : %s%n", result.getKind()); + System.out.printf(" file : %s (%,d bytes)%n", result.getFile(), result.getFileSizeBytes()); + System.out.printf(" cached : %s%n", result.isFromCache()); + if (result.getEmdbId() != null) { + System.out.printf(" EMDB : %s%n", result.getEmdbId()); + } + if (result.getRecommendedContourLevel() != null) { + System.out.printf(" contour : %s (author recommended)%n", result.getRecommendedContourLevel()); + } + if (result.getContourInSigma() != null) { + System.out.printf(" in sigma : %.2f%n", result.getContourInSigma()); + } + } catch (NoDensityMapException e) { + System.out.printf(" no map available%n"); + e.getAttempts().forEach((source, reason) -> + System.out.printf(" %-24s %s%n", source, reason)); + } + System.out.println(); + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/AbstractDensityMapProvider.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/AbstractDensityMapProvider.java new file mode 100644 index 0000000000..3ac4057ba2 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/AbstractDensityMapProvider.java @@ -0,0 +1,360 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.net.URLConnection; +import java.util.Date; + +import org.biojava.nbio.core.util.FileDownloadUtils; +import org.biojava.nbio.core.util.HttpStatusException; +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.io.LocalPDBDirectory; +import org.biojava.nbio.structure.io.LocalPDBDirectory.FetchBehavior; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Shared machinery for the concrete density map providers: cache lookup, + * download with validation, size limiting and format sanity checking. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public abstract class AbstractDensityMapProvider implements DensityMapProvider { + + private static final Logger logger = LoggerFactory.getLogger(AbstractDensityMapProvider.class); + + /** + * Anything smaller than this is not a usable map. Density servers answer with a + * short body rather than an error status when they have nothing, so a size + * floor is a necessary part of deciding whether a download succeeded. + */ + public static final long MIN_DENSITY_FILE_SIZE = 1024L; + + private File cacheRoot; + private FetchBehavior fetchBehavior = FetchBehavior.FETCH_FILES; + private long maxDownloadBytes = DensityMapCache.DEFAULT_MAX_DOWNLOAD_BYTES; + + /** + * @param cacheRoot the BioJava cache directory that the density directory sits in + */ + protected AbstractDensityMapProvider(File cacheRoot) { + this.cacheRoot = cacheRoot; + } + + /** @return the BioJava cache directory */ + public File getCacheRoot() { + return cacheRoot; + } + + /** @param cacheRoot the BioJava cache directory */ + public void setCacheRoot(File cacheRoot) { + this.cacheRoot = cacheRoot; + } + + /** @return how aggressively this provider re-fetches */ + public FetchBehavior getFetchBehavior() { + return fetchBehavior; + } + + /** @param fetchBehavior how aggressively to re-fetch */ + public void setFetchBehavior(FetchBehavior fetchBehavior) { + this.fetchBehavior = fetchBehavior == null ? FetchBehavior.FETCH_FILES : fetchBehavior; + } + + /** @return the download size limit in bytes, or 0 for no limit */ + public long getMaxDownloadBytes() { + return maxDownloadBytes; + } + + /** @param maxDownloadBytes the download size limit in bytes, or 0 for no limit */ + public void setMaxDownloadBytes(long maxDownloadBytes) { + this.maxDownloadBytes = maxDownloadBytes; + } + + @Override + public boolean supports(DensityMapKind kind) { + return kind != null && kind != DensityMapKind.AUTO; + } + + /** + * The identifier spelling to put into a URL: four characters and lower case + * where the entry has a short form. + * + * @param pdbId the entry + * @return the identifier for use in a URL + */ + protected String urlId(PdbId pdbId) { + return DensityCacheLayout.shortIdOrFull(pdbId).toLowerCase(); + } + + /** + * The effective fetch behaviour for a request, preferring the request's own + * setting over this provider's. + * + * @param request the request + * @return the behaviour to apply + */ + protected FetchBehavior effectiveFetchBehavior(DensityMapRequest request) { + return request.getFetchBehavior() == null ? fetchBehavior : request.getFetchBehavior(); + } + + /** + * The effective size limit for a request, preferring the request's own setting. + * + * @param request the request + * @return the limit in bytes, or 0 for no limit + */ + protected long effectiveMaxBytes(DensityMapRequest request) { + return request.getMaxDownloadBytes() < 0 ? maxDownloadBytes : request.getMaxDownloadBytes(); + } + + /** + * The cache directory to use for a request, preferring the request's override. + * + * @param request the request + * @return the cache directory + */ + protected File effectiveCacheRoot(DensityMapRequest request) { + return request.getCacheDir() == null ? cacheRoot : request.getCacheDir(); + } + + /** + * Obtains a map, serving it from the cache when the fetch behaviour permits and + * downloading it otherwise. + * + * @param request what was asked for + * @param url where to fetch from + * @param target where to cache it + * @param kind the concrete kind of map being fetched + * @param emdbId the EMDB entry, if this is an EM map; otherwise null + * @param contourLevel the author-recommended contour level, or null + * @param sigma the map RMS deviation, or null + * @return the result, or null if the source has nothing for this entry + * @throws IOException on transport failure + */ + protected DensityMapResult obtain(DensityMapRequest request, URL url, File target, DensityMapKind kind, + String emdbId, Double contourLevel, Double sigma) throws IOException { + + FetchBehavior behavior = effectiveFetchBehavior(request); + + if (isCacheUsable(target, url, behavior)) { + // The kind always comes from the request, never from the sidecar: a single + // cached file can hold more than one kind of map, so the sidecar's kind + // records what was asked for first, not what the caller wants now. + DensityMapResult cached = DensityMapResult.readMeta(target); + Double cachedContour = contourLevel != null || cached == null ? contourLevel + : cached.getRecommendedContourLevel(); + Double cachedSigma = sigma != null || cached == null ? sigma : cached.getSigma(); + DensityMapResult result = new DensityMapResult(target, getSource(), getFormat(), kind, + request.getPdbId(), emdbId, url.toString(), true, cachedContour, cachedSigma); + if (cached == null) { + // The file is good but its description was lost; write one rather than + // downloading several megabytes again. + result.writeMeta(); + } + return result; + } + + if (behavior == FetchBehavior.LOCAL_ONLY) { + throw new HttpStatusException(404, url.toString(), + "not in the local cache and downloads are disabled (FetchBehavior.LOCAL_ONLY)"); + } + + File dir = target.getAbsoluteFile().getParentFile(); + if (!dir.isDirectory() && !dir.mkdirs()) { + throw new IOException("Could not create density cache directory " + dir); + } + + enforceSizeLimit(url, effectiveMaxBytes(request), request); + + logger.info("Fetching {} density map for {} from {}", kind, + request.getPdbId() == null ? emdbId : request.getPdbId().getId(), url); + FileDownloadUtils.downloadFileWithValidation(url, target, null, FileDownloadUtils.Hash.UNKNOWN, + FileDownloadUtils.ETagPolicy.USE_IF_HEX_DIGEST); + + if (!isPlausibleMap(target)) { + // Do not leave a bad file behind to be picked up as a cache hit later. + deleteWithSidecars(target); + throw new IOException("The content downloaded from " + url + " is not a usable " + + getFormat() + " density map."); + } + + DensityMapResult result = new DensityMapResult(target, getSource(), getFormat(), kind, + request.getPdbId(), emdbId, url.toString(), false, contourLevel, sigma); + result.writeMeta(); + return result; + } + + /** + * Whether a cached file may be used as-is. + * + * @param target the cached file + * @param url where it came from + * @param behavior the fetch behaviour in force + * @return true if the cached file should be served + */ + protected boolean isCacheUsable(File target, URL url, FetchBehavior behavior) { + if (behavior == FetchBehavior.FORCE_DOWNLOAD) { + return false; + } + if (!target.isFile() || target.length() < MIN_DENSITY_FILE_SIZE) { + return false; + } + if (!FileDownloadUtils.validateFile(target)) { + logger.info("Cached density map [{}] failed validation and will be re-downloaded.", target); + return false; + } + if (!isPlausibleMap(target)) { + logger.info("Cached file [{}] is not a usable {} map and will be re-downloaded.", target, getFormat()); + return false; + } + if (behavior == FetchBehavior.LOCAL_ONLY) { + return true; + } + if (behavior == FetchBehavior.FETCH_IF_OUTDATED) { + Date serverDate = LocalPDBDirectory.getLastModifiedTime(url); + if (serverDate == null) { + // Density servers generate their responses on the fly and send no + // Last-Modified header. Treating that as "outdated" would re-download + // on every single call, so an unknown timestamp keeps the cache. + logger.debug("No server timestamp for {}; keeping the cached copy.", url); + return true; + } + return target.lastModified() >= serverDate.getTime(); + } + // FETCH_FILES, and FETCH_REMEDIATED which has no meaning for maps. + return true; + } + + /** + * Checks that a file looks like the format this provider delivers. Only the + * CCP4/MRC formats carry a recognisable stamp; the others are accepted on size + * alone. + * + * @param file the file to check + * @return true if the file is plausibly a map of this format + */ + protected boolean isPlausibleMap(File file) { + if (file == null || file.length() < MIN_DENSITY_FILE_SIZE) { + return false; + } + DensityFileFormat format = getFormat(); + if (format == DensityFileFormat.CCP4 || format == DensityFileFormat.CCP4_GZ) { + return Ccp4Header.isCcp4Quietly(file); + } + return true; + } + + /** + * Refuses a download whose declared size exceeds the limit, before any of the + * body is transferred. + * + * @param url the resource + * @param maxBytes the limit, or 0 for no limit + * @param request the request being served, used for reporting + * @throws DensityMapTooLargeException if the resource is too large + */ + protected void enforceSizeLimit(URL url, long maxBytes, DensityMapRequest request) + throws DensityMapTooLargeException { + if (maxBytes <= 0) { + return; + } + long size = declaredSize(url); + if (size > maxBytes) { + reportTooLarge(url, size, maxBytes, request); + throw new DensityMapTooLargeException(url.toString(), size, maxBytes); + } + } + + /** + * Announces that a map was skipped for being too large. + *

+ * This is reported through the logger and on both standard output and standard + * error. A user who asked for a map and silently got a coarser one from another + * source deserves to be told why, and a library log configuration that discards + * warnings should not be able to hide it. + * + * @param url the resource that was skipped + * @param size its size in bytes + * @param maxBytes the limit in bytes + * @param request the request being served + */ + protected void reportTooLarge(URL url, long size, long maxBytes, DensityMapRequest request) { + String entry = request.getPdbId() != null ? request.getPdbId().getId() + : (request.getEmdbId() != null ? request.getEmdbId() : "?"); + String message = String.format( + "BioJava density: skipping %s for %s (%,d bytes exceeds the %,d byte limit). " + + "Trying a smaller representation from another source; " + + "raise DensityMapCache.setMaxDownloadBytes() to allow it.", + url, entry, size, maxBytes); + logger.warn(message); + System.out.println(message); + System.err.println(message); + } + + /** + * The Content-Length a server declares for a resource. + * + * @param url the resource + * @return the size in bytes, or a negative value if the server did not say + */ + protected long declaredSize(URL url) { + try { + URLConnection connection = FileDownloadUtils.prepareURLConnection(url.toString(), 30000); + if (connection instanceof java.net.HttpURLConnection) { + ((java.net.HttpURLConnection) connection).setRequestMethod("HEAD"); + } + connection.connect(); + try { + return connection.getContentLengthLong(); + } finally { + if (connection instanceof java.net.HttpURLConnection) { + ((java.net.HttpURLConnection) connection).disconnect(); + } + } + } catch (IOException e) { + logger.debug("Could not determine the size of {}: {}", url, e.getMessage()); + return -1; + } + } + + /** + * Deletes a cached map along with its validation and metadata sidecars. + * + * @param target the cached map + */ + protected void deleteWithSidecars(File target) { + File dir = target.getAbsoluteFile().getParentFile(); + if (dir == null) { + return; + } + File[] siblings = dir.listFiles((d, name) -> name.startsWith(target.getName())); + if (siblings == null) { + return; + } + for (File f : siblings) { + if (!f.delete()) { + logger.debug("Could not delete [{}]", f); + } + } + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/Ccp4Header.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/Ccp4Header.java new file mode 100644 index 0000000000..594ad91927 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/Ccp4Header.java @@ -0,0 +1,133 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.zip.GZIPInputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Recognises CCP4/MRC map files by their header. + *

+ * This is a cheap but effective guard against a cached file that is not + * actually a map. A misbehaving or overloaded server can answer with an HTML + * error page and an HTTP 200, in which case nothing about the status code or the + * content length reveals the problem — but the missing MAP  + * stamp does, turning a silent failure into a clean cache miss. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class Ccp4Header { + + private static final Logger logger = LoggerFactory.getLogger(Ccp4Header.class); + + /** + * Byte offset of the four-character format stamp within a CCP4/MRC header. It + * sits in word 53 of the 256-word header. + */ + public static final int MAP_STAMP_OFFSET = 208; + + /** The stamp itself: the three letters of "MAP" followed by a space. */ + public static final String MAP_STAMP = "MAP "; + + /** Number of header bytes that must be readable for the check to be possible. */ + private static final int HEADER_BYTES = MAP_STAMP_OFFSET + 4; + + private Ccp4Header() { + } + + /** + * Checks whether a file is a CCP4/MRC map. Gzipped files are decompressed on + * the fly, so .map.gz works as well as .ccp4. + * + * @param file the file to check + * @return true if the CCP4 stamp is present + * @throws IOException if the file could not be read + */ + public static boolean isCcp4(File file) throws IOException { + if (file == null || !file.isFile()) { + return false; + } + // Deliberately no shortcut on file.length(): a gzipped map compresses to far + // less than the size of the header it contains, so a length test here would + // reject perfectly good small maps. Reading decides it instead. + try (InputStream in = openPossiblyGzipped(file)) { + return isCcp4(in); + } + } + + /** + * Checks whether a stream carries a CCP4/MRC map header. The stream is read + * from its current position and is not closed; it is not un-read afterwards, + * so pass a fresh stream or one that supports marking. + * + * @param in the stream to check, already decompressed + * @return true if the CCP4 stamp is present + * @throws IOException if the stream could not be read + */ + public static boolean isCcp4(InputStream in) throws IOException { + byte[] header = new byte[HEADER_BYTES]; + int read = 0; + while (read < HEADER_BYTES) { + int n = in.read(header, read, HEADER_BYTES - read); + if (n < 0) { + return false; // shorter than a CCP4 header, so certainly not one + } + read += n; + } + String stamp = new String(header, MAP_STAMP_OFFSET, 4, StandardCharsets.US_ASCII); + return MAP_STAMP.equals(stamp); + } + + /** + * Same as {@link #isCcp4(File)} but reports a problem rather than propagating + * it, for use in cache-validity checks where an unreadable file and an invalid + * one lead to the same action. + * + * @param file the file to check + * @return true if the file is readable and carries the CCP4 stamp + */ + public static boolean isCcp4Quietly(File file) { + try { + return isCcp4(file); + } catch (IOException e) { + logger.debug("Could not read [{}] to check for a CCP4 header: {}", file, e.getMessage()); + return false; + } + } + + private static InputStream openPossiblyGzipped(File file) throws IOException { + InputStream in = new BufferedInputStream(Files.newInputStream(file.toPath())); + in.mark(2); + int b1 = in.read(); + int b2 = in.read(); + in.reset(); + if (b1 == 0x1F && b2 == 0x8B) { + return new GZIPInputStream(in); + } + return in; + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityCacheLayout.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityCacheLayout.java new file mode 100644 index 0000000000..60e6ae7b61 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityCacheLayout.java @@ -0,0 +1,242 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.File; + +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.io.LocalPDBDirectory; + +/** + * Where cached density files live on disk. + *

+ * PDB-keyed maps follow the divided layout the rest of BioJava already uses, so + * that a cache with many entries does not end up with one enormous directory: + *

+ * <cache>/density/cb/1cbs_2fofc_pdbe.ccp4
+ * <cache>/density/cb/1cbs_fofc_pdbe.ccp4
+ * <cache>/density/cb/1cbs_2fofc_rcsbvs_d0.bcif
+ * <cache>/density/cb/1cbs_2fofc_wwpdb.cif.gz
+ * 
+ * Both the kind and the source appear in the file name, so no two combinations + * can collide. + *

+ * EMDB maps are keyed by EMDB identifier instead, mirroring the EMDB archive: + *

+ * <cache>/density/emd/EMD-0262/emd_0262.map.gz
+ * 
+ * Several PDB entries are often fitted into a single EM map, and those maps can + * be hundreds of megabytes, so keying them by PDB entry would cache the same + * enormous file many times over. + *

+ * The two-character directory needs no attention when the archive moves to + * extended identifiers in July 2027. It is a device for spreading files over + * directories, not a copy of the archive's own layout, and + * {@link LocalPDBDirectory#getMiddleHash(String)} counts from the right hand end + * of the identifier, so 1cbs and pdb_00001cbs both land + * in cb — which is also the rule the wwPDB documents for the + * new archive. + *

+ * Deliberately not the archive's per-entry layout. A cache is not a mirror + * and cannot become one: the archive publishes structure factors and map + * coefficients but never grids, so density has to be fetched whatever else is + * mirrored, and writing it into a directory whose contents are an exact copy of + * upstream puts it at the mercy of the next rsync --delete. Should + * that judgement ever be revisited, every cached path is computed here and + * nowhere else, so a different layout is a change to this class plus a fallback + * probe for files in the old places — not a cache that everyone has to + * discard. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class DensityCacheLayout { + + /** Name of the density sub-directory within the BioJava cache directory. */ + public static final String DENSITY_DIR = "density"; + + /** Name of the sub-directory holding EMDB-keyed maps. */ + public static final String EMDB_DIR = "emd"; + + /** Name of the sub-directory holding cached PDB-to-EMDB mappings. */ + public static final String EMDB_MAPPING_DIR = "emdb-mapping"; + + /** + * File-name token for a file that holds more than one kind of map, as a density + * server response does. + */ + public static final String BOTH_KINDS_TOKEN = "both"; + + private DensityCacheLayout() { + } + + /** + * The root density directory inside a cache directory. + * + * @param cacheRoot the BioJava cache directory + * @return the density directory, which need not exist yet + */ + public static File densityRoot(File cacheRoot) { + return new File(cacheRoot, DENSITY_DIR); + } + + /** + * The cache file for a PDB-keyed map. + * + * @param cacheRoot the BioJava cache directory + * @param pdbId the entry + * @param kind the kind of map + * @param source the service it came from + * @param format the file format + * @param qualifier an extra discriminator such as a detail level, or + * null. Anything that changes the content but not the + * entry, kind or source belongs here. + * @return the file, which need not exist + */ + public static File pdbMapFile(File cacheRoot, PdbId pdbId, DensityMapKind kind, DensityMapSource source, + DensityFileFormat format, String qualifier) { + return pdbMapFile(cacheRoot, pdbId, kind.getFileToken(), source, format, qualifier); + } + + /** + * The cache file for a PDB-keyed map, naming the kind explicitly. + *

+ * The separate kind token exists for sources that deliver more than one kind of + * map in a single file. A density server response, for instance, carries both + * the 2Fo-Fc and the Fo-Fc blocks, so it is cached once under + * {@link #BOTH_KINDS_TOKEN} rather than downloaded and stored twice. + * + * @param cacheRoot the BioJava cache directory + * @param pdbId the entry + * @param kindToken the token naming what the file holds + * @param source the service it came from + * @param format the file format + * @param qualifier an extra discriminator such as a detail level, or null + * @return the file, which need not exist + */ + public static File pdbMapFile(File cacheRoot, PdbId pdbId, String kindToken, DensityMapSource source, + DensityFileFormat format, String qualifier) { + String id = shortIdOrFull(pdbId).toLowerCase(); + File dir = new File(densityRoot(cacheRoot), LocalPDBDirectory.getMiddleHash(id)); + StringBuilder name = new StringBuilder(id) + .append('_').append(kindToken) + .append('_').append(source.getFileToken()); + if (qualifier != null && !qualifier.isEmpty()) { + name.append('_').append(qualifier); + } + name.append(format.getExtension()); + return new File(dir, name.toString()); + } + + /** + * The cache file for an EMDB-keyed map. + * + * @param cacheRoot the BioJava cache directory + * @param emdbId the EMDB entry, in any accepted form + * @param source the service it came from + * @param format the file format + * @param qualifier an extra discriminator such as a detail level, or null + * @return the file, which need not exist + */ + public static File emdbMapFile(File cacheRoot, String emdbId, DensityMapSource source, + DensityFileFormat format, String qualifier) { + String canonical = DensityMapRequest.normalizeEmdbId(emdbId); + String number = DensityMapRequest.emdbNumber(emdbId); + File dir = new File(new File(densityRoot(cacheRoot), EMDB_DIR), canonical); + StringBuilder name = new StringBuilder("emd_").append(number); + if (source != DensityMapSource.EMDB_MAP) { + name.append('_').append(source.getFileToken()); + } + if (qualifier != null && !qualifier.isEmpty()) { + name.append('_').append(qualifier); + } + name.append(format.getExtension()); + return new File(dir, name.toString()); + } + + /** + * The file caching the EMDB identifiers and author contour level associated + * with a PDB entry. + * + * @param cacheRoot the BioJava cache directory + * @param pdbId the entry + * @return the file, which need not exist + */ + public static File emdbMappingFile(File cacheRoot, PdbId pdbId) { + String id = shortIdOrFull(pdbId).toLowerCase(); + File dir = new File(new File(densityRoot(cacheRoot), EMDB_MAPPING_DIR), LocalPDBDirectory.getMiddleHash(id)); + return new File(dir, id + ".emdb.properties"); + } + + /** + * The file caching an EMDB entry's map metadata as served by the EMDB API. + * + * @param cacheRoot the BioJava cache directory + * @param emdbId the EMDB entry, in any accepted form + * @return the file, which need not exist + */ + public static File emdbMapInfoFile(File cacheRoot, String emdbId) { + String canonical = DensityMapRequest.normalizeEmdbId(emdbId); + File dir = new File(new File(densityRoot(cacheRoot), EMDB_DIR), canonical); + return new File(dir, canonical + ".map-info.json"); + } + + /** + * The companion name a density-server file must have for Jmol to read its + * difference-map block. + *

+ * A density server response holds both a 2FO-FC and an + * FO-FC data block, and Jmol's reader chooses between them by + * testing whether the file name contains the literal text + * &diff=1. That works for a URL fetched straight from the + * server, where the marker rides along in the query string, but a cached local + * file has no query string; appending the marker to the file URL only makes + * Jmol look for a file that does not exist. Putting the marker into the name + * itself is what actually selects the block. + *

+ * This was verified against Jmol 14.31.10 and is unchanged in current Jmol: the + * relevant line in BCifDensityReader still carries the author's + * "what about cached data" to-do beside it. Should Jmol gain a cleaner way to + * choose the block, this can be retired. + * + * @param mapFile the cached density-server file + * @return the sibling path that selects the difference map + */ + public static File differenceMarkerFile(File mapFile) { + String name = mapFile.getName(); + int dot = name.lastIndexOf('.'); + String stem = dot < 0 ? name : name.substring(0, dot); + String ext = dot < 0 ? "" : name.substring(dot); + return new File(mapFile.getAbsoluteFile().getParentFile(), stem + "&diff=1" + ext); + } + + /** + * The short four-character spelling of an identifier where one exists. + *

+ * Every entry in the archive today has a four-character form, and the density + * services accept only that spelling. Extended-only identifiers will appear + * eventually; rather than refusing them, the extended spelling is passed + * through so that the services can start accepting it without a change here. + * + * @param pdbId the identifier + * @return the short spelling if available, otherwise the full one + */ + public static String shortIdOrFull(PdbId pdbId) { + return pdbId.getId(true); + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityFileFormat.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityFileFormat.java new file mode 100644 index 0000000000..11cbb0e27e --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityFileFormat.java @@ -0,0 +1,89 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +/** + * The file format a density map was delivered in. + *

+ * The important distinction here is {@link #isJmolLoadable()}. Most of these + * formats are sampled grids that a viewer can contour directly; map + * coefficients are not, they are structure factors that require a Fourier + * transform first. Handing the latter to a viewer produces nothing at all, so + * the difference has to be visible to callers. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public enum DensityFileFormat { + + /** A CCP4/MRC sampled map, as served pre-computed by PDBe. */ + CCP4(".ccp4", true), + + /** A gzipped CCP4/MRC map, the form EMDB distributes its primary maps in. */ + CCP4_GZ(".map.gz", true), + + /** A BinaryCIF volume slice from a Mol* density server. */ + BCIF_VOLUME(".bcif", true), + + /** A text CIF volume slice from a Mol* density server. */ + CIF_VOLUME(".cif", true), + + /** + * Structure-factor amplitudes and phases in mmCIF, as published with the wwPDB + * validation reports. Not a density grid. A Fourier transform (for + * example gemmi sf2map, or CCP4's fft after + * cif2mtz) is needed before these can be displayed. + */ + MAP_COEFFICIENTS_CIF_GZ(".cif.gz", false); + + private final String extension; + private final boolean jmolLoadable; + + DensityFileFormat(String extension, boolean jmolLoadable) { + this.extension = extension; + this.jmolLoadable = jmolLoadable; + } + + /** + * @return the file extension used for cached files of this format, including + * the leading dot + */ + public String getExtension() { + return extension; + } + + /** + * Whether a viewer can contour this file as it stands. + * + * @return false for {@link #MAP_COEFFICIENTS_CIF_GZ}, which needs + * an FFT first; true for the sampled grid formats + */ + public boolean isJmolLoadable() { + return jmolLoadable; + } + + /** + * Whether files of this format are gzip-compressed on disk. Jmol detects gzip + * from the magic bytes, so such files do not need decompressing before display. + * + * @return true for the compressed formats + */ + public boolean isCompressed() { + return this == CCP4_GZ || this == MAP_COEFFICIENTS_CIF_GZ; + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapCache.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapCache.java new file mode 100644 index 0000000000..adbbd917a8 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapCache.java @@ -0,0 +1,526 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.biojava.nbio.core.util.FileDownloadUtils; +import org.biojava.nbio.core.util.HttpStatusException; +import org.biojava.nbio.structure.ExperimentalTechnique; +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.Structure; +import org.biojava.nbio.structure.align.util.UserConfiguration; +import org.biojava.nbio.structure.io.LocalPDBDirectory.FetchBehavior; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Downloads and caches electron density and cryo-EM maps, trying several sources + * in turn until one produces a map. + *

+ * Cached files live under the BioJava cache directory (PDB_CACHE_DIR), + * laid out as described in {@link DensityCacheLayout}. Typical use is simply: + *

+ * DensityMapCache cache = new DensityMapCache();
+ * DensityMapResult map = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
+ * File file = map.getFile();
+ * 
+ *

+ * Source order. Sources are tried smallest-adequate-first, because they + * differ enormously in size for the same entry and the smallest is usually + * perfectly adequate to look at. For 1cbs a density-server slice is roughly a + * tenth the size of the equivalent pair of CCP4 files; for the cryo-EM entry + * behind EMD-0262 it is a few hundred kilobytes against 106 MB. Anything + * that cannot be displayed at all is tried last, and + * {@link DensityMapSource#WWPDB_MAP_COEFFICIENTS} is disabled altogether by + * default for that reason. Override with {@link #setSourceChain(DensityMapKind, + * java.util.List)} or {@link #setSourceEnabled(DensityMapSource, boolean)}. + *

+ * Failures. A source that has nothing for an entry is skipped and the next + * is tried; if every source is exhausted, {@link NoDensityMapException} is thrown + * carrying the reason from each one, so a caller can explain what happened. + * Genuine transport failures abort the chain instead, so that a network outage is + * never reported as "this entry has no density". + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class DensityMapCache { + + private static final Logger logger = LoggerFactory.getLogger(DensityMapCache.class); + + /** + * Default ceiling on a single download, 256 MiB. Exceeding it is not an + * error: the chain moves on to a source that offers a smaller representation. + * Set to 0 to remove the limit. + */ + public static final long DEFAULT_MAX_DOWNLOAD_BYTES = 256L * 1024 * 1024; + + /** Order in which sources are tried for X-ray and neutron entries. */ + public static final List DEFAULT_XRAY_SOURCE_CHAIN = Collections.unmodifiableList(Arrays.asList( + DensityMapSource.RCSB_VOLUME_SERVER, + DensityMapSource.PDBE_CCP4, + DensityMapSource.PDBE_VOLUME_SERVER, + DensityMapSource.WWPDB_MAP_COEFFICIENTS)); + + /** Order in which sources are tried for cryo-EM entries. */ + public static final List DEFAULT_EM_SOURCE_CHAIN = Collections.unmodifiableList(Arrays.asList( + DensityMapSource.RCSB_VOLUME_SERVER, + DensityMapSource.PDBE_VOLUME_SERVER, + DensityMapSource.EMDB_MAP)); + + private static DensityMapCache instance; + + private File cacheRoot; + private FetchBehavior fetchBehavior = FetchBehavior.FETCH_FILES; + private long maxDownloadBytes = DEFAULT_MAX_DOWNLOAD_BYTES; + private final Map providers = new EnumMap<>(DensityMapSource.class); + private final Set disabled = EnumSet.of(DensityMapSource.WWPDB_MAP_COEFFICIENTS); + private List xraySourceChain = DEFAULT_XRAY_SOURCE_CHAIN; + private List emSourceChain = DEFAULT_EM_SOURCE_CHAIN; + private EmdbEntryResolver emdbResolver; + + /** + * Creates a cache using the standard BioJava cache directory, i.e. + * PDB_CACHE_DIR falling back to PDB_DIR. + */ + public DensityMapCache() { + this(new UserConfiguration().getCacheFilePath()); + } + + /** + * @param cachePath the directory to cache under + */ + public DensityMapCache(String cachePath) { + this.cacheRoot = new File(FileDownloadUtils.expandUserHome(cachePath)); + this.emdbResolver = new EmdbEntryResolver(cacheRoot); + buildDefaultProviders(); + } + + /** + * A lazily created shared instance, for callers that do not want to manage one. + * + * @return the shared cache + */ + public static synchronized DensityMapCache getInstance() { + if (instance == null) { + instance = new DensityMapCache(); + } + return instance; + } + + private void buildDefaultProviders() { + providers.clear(); + register(new VolumeServerProvider(cacheRoot, VolumeServerProvider.Host.RCSB)); + register(new VolumeServerProvider(cacheRoot, VolumeServerProvider.Host.PDBE)); + register(new PdbeCcp4MapProvider(cacheRoot)); + register(new EmdbMapProvider(cacheRoot, emdbResolver)); + register(new WwpdbMapCoefficientsProvider(cacheRoot)); + applySettingsToProviders(); + } + + private void register(DensityMapProvider provider) { + providers.put(provider.getSource(), provider); + } + + private void applySettingsToProviders() { + for (DensityMapProvider p : providers.values()) { + if (p instanceof AbstractDensityMapProvider) { + AbstractDensityMapProvider a = (AbstractDensityMapProvider) p; + a.setCacheRoot(cacheRoot); + a.setFetchBehavior(fetchBehavior); + a.setMaxDownloadBytes(maxDownloadBytes); + } + } + emdbResolver.setCacheRoot(cacheRoot); + emdbResolver.setFetchBehavior(fetchBehavior); + } + + /** @return the directory maps are cached under */ + public String getCachePath() { + return cacheRoot.getAbsolutePath(); + } + + /** @param cachePath the directory to cache under */ + public void setCachePath(String cachePath) { + this.cacheRoot = new File(FileDownloadUtils.expandUserHome(cachePath)); + applySettingsToProviders(); + } + + /** @return how aggressively cached maps are re-fetched */ + public FetchBehavior getFetchBehavior() { + return fetchBehavior; + } + + /** + * @param fetchBehavior how aggressively to re-fetch. {@link FetchBehavior#FETCH_REMEDIATED} + * behaves as {@link FetchBehavior#FETCH_FILES}: the 2011 remediation date + * is a coordinate-file concept with no meaning for maps. + */ + public void setFetchBehavior(FetchBehavior fetchBehavior) { + this.fetchBehavior = fetchBehavior == null ? FetchBehavior.FETCH_FILES : fetchBehavior; + applySettingsToProviders(); + } + + /** @return the ceiling on a single download in bytes, or 0 for no limit */ + public long getMaxDownloadBytes() { + return maxDownloadBytes; + } + + /** @param maxDownloadBytes the ceiling on a single download in bytes, or 0 for no limit */ + public void setMaxDownloadBytes(long maxDownloadBytes) { + this.maxDownloadBytes = maxDownloadBytes; + applySettingsToProviders(); + } + + /** + * The order sources are tried in for a given kind of map. + * + * @param kind the kind of map + * @return the source order + */ + public List getSourceChain(DensityMapKind kind) { + return kind == DensityMapKind.EM ? emSourceChain : xraySourceChain; + } + + /** + * Overrides the order sources are tried in. + * + * @param kind {@link DensityMapKind#EM} to set the cryo-EM order, anything else + * to set the X-ray order + * @param chain the new order + */ + public void setSourceChain(DensityMapKind kind, List chain) { + List copy = Collections.unmodifiableList(new ArrayList<>(chain)); + if (kind == DensityMapKind.EM) { + emSourceChain = copy; + } else { + xraySourceChain = copy; + } + } + + /** + * @param source the source + * @return whether it will be tried + */ + public boolean isSourceEnabled(DensityMapSource source) { + return !disabled.contains(source); + } + + /** + * Enables or disables a source. + *

+ * {@link DensityMapSource#WWPDB_MAP_COEFFICIENTS} is disabled by default + * because it delivers structure factors rather than a map; enable it explicitly + * if you want the archival form and are prepared to run a Fourier transform. + * + * @param source the source + * @param enabled whether to try it + */ + public void setSourceEnabled(DensityMapSource source, boolean enabled) { + if (enabled) { + disabled.remove(source); + } else { + disabled.add(source); + } + } + + /** + * Replaces the provider used for a source. Mainly a testing seam, but also the + * way to plug in a site-local mirror. + * + * @param provider the provider to use + */ + public void registerProvider(DensityMapProvider provider) { + register(provider); + applySettingsToProviders(); + } + + /** @return the resolver used to map PDB entries to EMDB entries */ + public EmdbEntryResolver getEmdbResolver() { + return emdbResolver; + } + + /** @param resolver the resolver used to map PDB entries to EMDB entries */ + public void setEmdbResolver(EmdbEntryResolver resolver) { + this.emdbResolver = resolver; + applySettingsToProviders(); + } + + /** + * Fetches a density map, using the cache where possible. + * + * @param pdbId the entry + * @param kind the kind of map, or {@link DensityMapKind#AUTO} to take whatever + * the entry has + * @return the map + * @throws NoDensityMapException if no enabled source has a map for this entry + * @throws IOException on transport failure + */ + public DensityMapResult getDensityMap(PdbId pdbId, DensityMapKind kind) throws IOException { + return getDensityMap(DensityMapRequest.builder(pdbId).kind(kind).build()); + } + + /** + * Fetches a density map for a PDB or EMDB identifier. + * + * @param id a PDB identifier, or an EMDB identifier such as EMD-0262 + * @param kind the kind of map wanted + * @return the map + * @throws NoDensityMapException if no enabled source has a map for this entry + * @throws IOException on transport failure + */ + public DensityMapResult getDensityMap(String id, DensityMapKind kind) throws IOException { + return getDensityMap(DensityMapRequest.builder(id).kind(kind).build()); + } + + /** + * Fetches a density map for an already-loaded structure. + *

+ * Knowing the structure lets the experimental method be read directly rather + * than guessed, so a cryo-EM entry goes straight to the EM sources instead of + * trying the X-ray ones first. No extra network request is involved. + * + * @param structure the structure + * @param kind the kind of map wanted + * @return the map + * @throws NoDensityMapException if no enabled source has a map for this entry + * @throws IOException on transport failure + */ + public DensityMapResult getDensityMap(Structure structure, DensityMapKind kind) throws IOException { + PdbId pdbId = structure.getPdbId(); + if (pdbId == null) { + throw new IOException("The structure has no PDB ID, so no density map can be looked up for it."); + } + DensityMapRequest request = DensityMapRequest.builder(pdbId).kind(kind).build(); + return getDensityMap(request, kindOrderFor(structure, kind)); + } + + /** + * Fetches a density map. + * + * @param request what is wanted + * @return the map + * @throws NoDensityMapException if no enabled source has a map for this entry + * @throws IOException on transport failure + */ + public DensityMapResult getDensityMap(DensityMapRequest request) throws IOException { + return getDensityMap(request, request.getKind().resolve()); + } + + /** + * As {@link #getDensityMap(PdbId, DensityMapKind)} but returning an empty + * {@link Optional} rather than throwing when nothing is available. Genuine + * transport failures are still logged and swallowed, so use this only where + * "no map" and "could not reach the server" need not be told apart. + * + * @param pdbId the entry + * @param kind the kind of map wanted + * @return the map, if one could be obtained + */ + public Optional findDensityMap(PdbId pdbId, DensityMapKind kind) { + try { + return Optional.of(getDensityMap(pdbId, kind)); + } catch (NoDensityMapException e) { + logger.debug("{}", e.getMessage()); + return Optional.empty(); + } catch (IOException e) { + logger.warn("Could not fetch a density map for {}: {}", pdbId.getId(), e.getMessage()); + return Optional.empty(); + } + } + + /** + * Fetches both halves of the conventional X-ray pair: the 2mFo-DFc map and the + * mFo-DFc difference map. + * + * @param pdbId the entry + * @return whichever of the two could be obtained, in that order; possibly empty + */ + public List getDifferenceMapPair(PdbId pdbId) { + List results = new ArrayList<>(2); + findDensityMap(pdbId, DensityMapKind.TWO_FO_FC).ifPresent(results::add); + findDensityMap(pdbId, DensityMapKind.FO_FC).ifPresent(results::add); + return results; + } + + /** + * Looks a map up in the cache without contacting any server. + * + * @param pdbId the entry + * @param kind the kind of map + * @param source the source it would have come from + * @return the cached map, or null if it is not cached + */ + public DensityMapResult getCached(PdbId pdbId, DensityMapKind kind, DensityMapSource source) { + DensityMapProvider provider = providers.get(source); + if (provider == null) { + return null; + } + File file = DensityCacheLayout.pdbMapFile(cacheRoot, pdbId, kind, source, provider.getFormat(), null); + return file.isFile() ? DensityMapResult.readMeta(file) : null; + } + + /** + * Removes every cached density file for an entry, including the sidecars. + * + * @param pdbId the entry + * @return the number of files deleted + */ + public int deleteDensityMaps(PdbId pdbId) { + String id = DensityCacheLayout.shortIdOrFull(pdbId).toLowerCase(); + File dir = DensityCacheLayout.pdbMapFile(cacheRoot, pdbId, DensityMapKind.TWO_FO_FC, + DensityMapSource.PDBE_CCP4, DensityFileFormat.CCP4, null).getParentFile(); + File[] files = dir == null ? null : dir.listFiles((d, name) -> name.startsWith(id + "_")); + if (files == null) { + return 0; + } + int deleted = 0; + for (File f : files) { + if (f.delete()) { + deleted++; + } + } + return deleted; + } + + private DensityMapResult getDensityMap(DensityMapRequest request, List kinds) throws IOException { + Map attempts = new LinkedHashMap<>(); + + for (DensityMapKind kind : kinds) { + DensityMapRequest kindRequest = request.withKind(kind); + + if (kind == DensityMapKind.EM && kindRequest.getEmdbId() == null) { + String emdbId = resolveEmdbId(kindRequest); + if (emdbId == null) { + attempts.put(DensityMapSource.EMDB_MAP, "no associated EMDB entry"); + continue; + } + kindRequest = kindRequest.withEmdbId(emdbId); + } + + List chain = kindRequest.getSourceChain() != null + ? kindRequest.getSourceChain() : getSourceChain(kind); + + for (DensityMapSource source : chain) { + if (!isSourceEnabled(source)) { + attempts.put(source, "disabled"); + continue; + } + DensityMapProvider provider = providers.get(source); + if (provider == null) { + attempts.put(source, "no provider registered"); + continue; + } + if (!provider.supports(kind)) { + continue; // structurally impossible; not worth reporting + } + if (!kindRequest.isAllowNonRenderableFormats() && !provider.getFormat().isJmolLoadable()) { + attempts.put(source, "cannot be displayed without a Fourier transform"); + continue; + } + + try { + DensityMapResult result = provider.fetch(kindRequest); + if (result != null) { + return withContourLevel(result); + } + attempts.put(source, "no map for this entry"); + } catch (HttpStatusException e) { + if (!e.isNotFound()) { + // A server error or an authentication problem is a real failure, + // not evidence that the entry has no density. + throw e; + } + attempts.put(source, "HTTP " + e.getStatusCode()); + } catch (DensityMapTooLargeException e) { + attempts.put(source, "too large (" + e.getSizeBytes() + " bytes)"); + } catch (IOException e) { + throw new IOException("Failed to fetch a density map for " + + (request.getPdbId() == null ? request.getEmdbId() : request.getPdbId().getId()) + + " from " + source + ": " + e.getMessage(), e); + } + } + } + + throw new NoDensityMapException(request.getPdbId(), request.getKind(), attempts); + } + + /** + * Fills in the author-recommended contour level for an EM map when the source + * that supplied the file did not know it. + *

+ * A density server returns voxels and nothing else, but an EM map is + * conventionally displayed at the level its depositors chose rather than at a + * multiple of sigma, so a viewer needs that number whichever source the map came + * from. It costs one small metadata request, cached thereafter. + */ + private DensityMapResult withContourLevel(DensityMapResult result) { + if (result.getKind() != DensityMapKind.EM || result.getRecommendedContourLevel() != null + || result.getEmdbId() == null || emdbResolver == null) { + return result; + } + EmdbEntryInfo info = emdbResolver.getEntryInfo(result.getEmdbId()); + if (info == null || (info.getRecommendedContourLevel() == null && info.getSigma() == null)) { + return result; + } + DensityMapResult enriched = new DensityMapResult(result.getFile(), result.getSource(), result.getFormat(), + result.getKind(), result.getPdbId(), result.getEmdbId(), result.getSourceUrl(), result.isFromCache(), + info.getRecommendedContourLevel(), info.getSigma()); + enriched.writeMeta(); + return enriched; + } + + private String resolveEmdbId(DensityMapRequest request) { + if (request.getPdbId() == null || emdbResolver == null) { + return null; + } + List ids = emdbResolver.getEmdbIds(request.getPdbId()); + return ids.isEmpty() ? null : ids.get(0); + } + + /** + * Chooses which kinds to try, and in what order, using the structure's declared + * experimental method. Reading it from the structure costs nothing; note that + * the resolution field is deliberately not consulted, since BioJava parses it + * incorrectly for some cryo-EM entries (biojava/biojava#1000). + */ + private List kindOrderFor(Structure structure, DensityMapKind kind) { + if (kind != DensityMapKind.AUTO) { + return kind.resolve(); + } + Set techniques = structure.getPDBHeader() == null + ? null : structure.getPDBHeader().getExperimentalTechniques(); + if (techniques != null && techniques.contains(ExperimentalTechnique.ELECTRON_MICROSCOPY) + && !ExperimentalTechnique.isCrystallographic(techniques)) { + return Arrays.asList(DensityMapKind.EM, DensityMapKind.TWO_FO_FC); + } + return kind.resolve(); + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapKind.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapKind.java new file mode 100644 index 0000000000..8b472092ef --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapKind.java @@ -0,0 +1,99 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * The kind of density map being requested, i.e. what the values in the grid mean. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public enum DensityMapKind { + + /** + * The 2mFo-DFc "best" map: the electron density itself, showing how the model + * fits the experimental data. This is what is normally meant by "the electron + * density" of an X-ray structure. + */ + TWO_FO_FC("2fofc"), + + /** + * The mFo-DFc difference map, showing density that the model does not account + * for (positive) and model that has no density to support it (negative). It is + * conventionally displayed as a signed pair of surfaces. + */ + FO_FC("fofc"), + + /** + * The primary map of a cryo-EM or cryo-ET reconstruction. Strictly this is a + * Coulomb potential map rather than an electron density map, and it is + * conventionally contoured at an absolute author-recommended level rather than + * in multiples of sigma. + */ + EM("em"), + + /** + * Not a map in itself: asks for whichever map the entry actually has. Resolves + * to {@link #TWO_FO_FC} first and then {@link #EM}, which covers X-ray and + * cryo-EM entries without the caller having to know which it is holding. + */ + AUTO(null); + + private final String fileToken; + + DensityMapKind(String fileToken) { + this.fileToken = fileToken; + } + + /** + * The short token used to distinguish this kind in a cached file name. + * + * @return the token, or null for {@link #AUTO}, which is never + * itself cached + */ + public String getFileToken() { + return fileToken; + } + + /** + * Expands this kind into the concrete kinds to try, in order. + * + * @return a single-element list for a concrete kind, or the X-ray-then-EM order + * for {@link #AUTO} + */ + public List resolve() { + if (this == AUTO) { + return Arrays.asList(TWO_FO_FC, EM); + } + return Collections.singletonList(this); + } + + /** + * Whether this kind is conventionally displayed as a signed pair of surfaces, + * one positive and one negative. + * + * @return true only for {@link #FO_FC} + */ + public boolean isDifferenceMap() { + return this == FO_FC; + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapProvider.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapProvider.java new file mode 100644 index 0000000000..c628387ffd --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapProvider.java @@ -0,0 +1,78 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.IOException; + +import org.biojava.nbio.core.util.HttpStatusException; + +/** + * Fetches density maps from one particular service. + *

+ * Implementations are combined into an ordered chain by {@link DensityMapCache}, + * which tries each in turn until one produces a map. The contract around + * exceptions is what makes that chain safe: + *

+ * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public interface DensityMapProvider { + + /** + * @return which service this provider talks to + */ + DensityMapSource getSource(); + + /** + * @return the format this provider delivers + */ + DensityFileFormat getFormat(); + + /** + * Whether this provider can serve a given kind of map at all. Used to skip + * requests that could not possibly succeed, such as asking an X-ray map service + * for a cryo-EM map. + * + * @param kind the kind of map wanted + * @return true if it is worth trying + */ + boolean supports(DensityMapKind kind); + + /** + * Fetches a map, using the cache if the request's fetch behaviour allows. + * + * @param request what is wanted. Its kind is always concrete, never + * {@link DensityMapKind#AUTO}. + * @return the map, or null if this service has nothing for the entry + * @throws DensityMapTooLargeException if the map exceeds the request's size limit + * @throws IOException on transport failure; use {@link HttpStatusException} so + * that a missing resource can be told apart from a broken connection + */ + DensityMapResult fetch(DensityMapRequest request) throws IOException; +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapRequest.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapRequest.java new file mode 100644 index 0000000000..d6677a3835 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapRequest.java @@ -0,0 +1,275 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.File; +import java.util.Collections; +import java.util.List; + +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.io.LocalPDBDirectory.FetchBehavior; + +/** + * A request for a density map, describing what is wanted and how hard to look + * for it. + *

+ * Instances are immutable; build them with {@link #builder(PdbId)} or + * {@link #builder(String)}. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class DensityMapRequest { + + private final PdbId pdbId; + private final String emdbId; + private final DensityMapKind kind; + private final FetchBehavior fetchBehavior; + private final File cacheDir; + private final boolean allowNonRenderableFormats; + private final long maxDownloadBytes; + private final List sourceChain; + + private DensityMapRequest(Builder b) { + this.pdbId = b.pdbId; + this.emdbId = b.emdbId; + this.kind = b.kind; + this.fetchBehavior = b.fetchBehavior; + this.cacheDir = b.cacheDir; + this.allowNonRenderableFormats = b.allowNonRenderableFormats; + this.maxDownloadBytes = b.maxDownloadBytes; + this.sourceChain = b.sourceChain == null ? null : Collections.unmodifiableList(b.sourceChain); + } + + /** + * Starts a request for a PDB entry. + * + * @param pdbId the entry + * @return a new builder + */ + public static Builder builder(PdbId pdbId) { + return new Builder(pdbId, null); + } + + /** + * Starts a request from an identifier string. An identifier beginning with + * EMD- (case-insensitively) is taken as an EMDB entry, anything + * else as a PDB entry. + * + * @param id a PDB or EMDB identifier + * @return a new builder + */ + public static Builder builder(String id) { + if (id == null) { + throw new IllegalArgumentException("Identifier must not be null"); + } + String trimmed = id.trim(); + if (trimmed.toUpperCase().startsWith("EMD-") || trimmed.toUpperCase().startsWith("EMD_")) { + return new Builder(null, normalizeEmdbId(trimmed)).kind(DensityMapKind.EM); + } + return new Builder(new PdbId(trimmed), null); + } + + /** + * Normalises an EMDB identifier to the canonical EMD-1234 form. + * + * @param emdbId an identifier such as emd-1234, EMD_1234 + * or a bare number + * @return the canonical form + */ + public static String normalizeEmdbId(String emdbId) { + if (emdbId == null) { + return null; + } + String digits = emdbId.trim().toUpperCase().replaceFirst("^EMD[-_]?", ""); + return "EMD-" + digits; + } + + /** + * Extracts the numeric part of an EMDB identifier, as used in file names and + * some URL templates. + * + * @param emdbId an EMDB identifier in any accepted form + * @return the digits, without the EMD- prefix + */ + public static String emdbNumber(String emdbId) { + return normalizeEmdbId(emdbId).substring("EMD-".length()); + } + + /** @return the PDB entry requested, or null for an EMDB-only request */ + public PdbId getPdbId() { + return pdbId; + } + + /** @return the EMDB entry, if known or explicitly requested; otherwise null */ + public String getEmdbId() { + return emdbId; + } + + /** @return the kind of map wanted; never null */ + public DensityMapKind getKind() { + return kind; + } + + /** @return how aggressively to re-fetch, or null to use the cache's setting */ + public FetchBehavior getFetchBehavior() { + return fetchBehavior; + } + + /** @return an override for the cache directory, or null to use the cache's own */ + public File getCacheDir() { + return cacheDir; + } + + /** + * Whether sources that deliver something a viewer cannot contour directly + * — currently only map coefficients — may be used. + * + * @return true by default; a viewer should set it to + * false + */ + public boolean isAllowNonRenderableFormats() { + return allowNonRenderableFormats; + } + + /** @return the download size limit in bytes, or 0 for no limit */ + public long getMaxDownloadBytes() { + return maxDownloadBytes; + } + + /** @return an explicit source order, or null to let the cache choose */ + public List getSourceChain() { + return sourceChain; + } + + /** + * Returns a copy of this request with a different map kind, used when expanding + * {@link DensityMapKind#AUTO}. + * + * @param newKind the kind to use + * @return a new request + */ + public DensityMapRequest withKind(DensityMapKind newKind) { + return toBuilder().kind(newKind).build(); + } + + /** + * Returns a copy of this request with the EMDB entry filled in. + * + * @param newEmdbId the EMDB identifier + * @return a new request + */ + public DensityMapRequest withEmdbId(String newEmdbId) { + return toBuilder().emdbId(newEmdbId).build(); + } + + private Builder toBuilder() { + Builder b = new Builder(pdbId, emdbId); + b.kind = kind; + b.fetchBehavior = fetchBehavior; + b.cacheDir = cacheDir; + b.allowNonRenderableFormats = allowNonRenderableFormats; + b.maxDownloadBytes = maxDownloadBytes; + b.sourceChain = sourceChain; + return b; + } + + @Override + public String toString() { + return String.format("DensityMapRequest[%s%s, %s]", + pdbId == null ? "" : pdbId.getId(), + emdbId == null ? "" : (pdbId == null ? emdbId : "/" + emdbId), + kind); + } + + /** + * Builder for {@link DensityMapRequest}. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ + public static final class Builder { + + private final PdbId pdbId; + private String emdbId; + private DensityMapKind kind = DensityMapKind.AUTO; + private FetchBehavior fetchBehavior; + private File cacheDir; + private boolean allowNonRenderableFormats = true; + private long maxDownloadBytes = -1; + private List sourceChain; + + private Builder(PdbId pdbId, String emdbId) { + this.pdbId = pdbId; + this.emdbId = emdbId; + } + + /** @param kind the kind of map wanted; null means {@link DensityMapKind#AUTO} */ + public Builder kind(DensityMapKind kind) { + this.kind = kind == null ? DensityMapKind.AUTO : kind; + return this; + } + + /** @param emdbId the EMDB entry to use, skipping the PDB-to-EMDB lookup */ + public Builder emdbId(String emdbId) { + this.emdbId = emdbId == null ? null : normalizeEmdbId(emdbId); + return this; + } + + /** @param fetchBehavior how aggressively to re-fetch */ + public Builder fetchBehavior(FetchBehavior fetchBehavior) { + this.fetchBehavior = fetchBehavior; + return this; + } + + /** @param cacheDir an override for the cache directory */ + public Builder cacheDir(File cacheDir) { + this.cacheDir = cacheDir; + return this; + } + + /** + * @param allow whether formats that cannot be contoured directly may be used. + * A viewer should pass false. + */ + public Builder allowNonRenderableFormats(boolean allow) { + this.allowNonRenderableFormats = allow; + return this; + } + + /** @param bytes the download size limit, or 0 for no limit, or a negative value to use the cache's setting */ + public Builder maxDownloadBytes(long bytes) { + this.maxDownloadBytes = bytes; + return this; + } + + /** @param chain an explicit source order, overriding the cache's choice */ + public Builder sourceChain(List chain) { + this.sourceChain = chain; + return this; + } + + /** @return the finished request */ + public DensityMapRequest build() { + if (pdbId == null && emdbId == null) { + throw new IllegalArgumentException("A request needs either a PDB ID or an EMDB ID"); + } + return new DensityMapRequest(this); + } + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapResult.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapResult.java new file mode 100644 index 0000000000..7275466c9e --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapResult.java @@ -0,0 +1,282 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.time.Instant; +import java.util.Properties; + +import org.biojava.nbio.structure.PdbId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A density map that was successfully obtained, together with everything a + * caller needs to know in order to use it. + *

+ * Which source answered matters to the caller and is therefore part of the + * result: the file might be a full CCP4 grid, a downsampled volume slice, or + * — if non-renderable formats were allowed — a set of structure + * factors that must be Fourier-transformed before anything can be drawn. See + * {@link #isRenderable()}. + *

+ * Every field is persisted to a .meta sidecar beside the cached + * file, so a result can be reconstructed later without contacting any server. + * That is what lets the cache serve + * {@link org.biojava.nbio.structure.io.LocalPDBDirectory.FetchBehavior#LOCAL_ONLY} + * requests fully offline. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class DensityMapResult { + + private static final Logger logger = LoggerFactory.getLogger(DensityMapResult.class); + + /** Extension of the metadata sidecar written beside every cached map. */ + public static final String META_EXT = ".meta"; + + private final File file; + private final DensityMapSource source; + private final DensityFileFormat format; + private final DensityMapKind kind; + private final PdbId pdbId; + private final String emdbId; + private final String sourceUrl; + private final boolean fromCache; + private final Double recommendedContourLevel; + private final Double sigma; + + /** + * @param file the cached map file + * @param source which service supplied it + * @param format the file format + * @param kind what the values mean; never {@link DensityMapKind#AUTO} + * @param pdbId the PDB entry, may be null + * @param emdbId the EMDB entry, may be null + * @param sourceUrl the URL it came from + * @param fromCache whether it was already on disk rather than freshly downloaded + * @param recommendedContourLevel the author-recommended contour level in absolute + * map units, or null if unknown + * @param sigma the RMS deviation of the map, or null if unknown + */ + public DensityMapResult(File file, DensityMapSource source, DensityFileFormat format, DensityMapKind kind, + PdbId pdbId, String emdbId, String sourceUrl, boolean fromCache, + Double recommendedContourLevel, Double sigma) { + this.file = file; + this.source = source; + this.format = format; + this.kind = kind; + this.pdbId = pdbId; + this.emdbId = emdbId; + this.sourceUrl = sourceUrl; + this.fromCache = fromCache; + this.recommendedContourLevel = recommendedContourLevel; + this.sigma = sigma; + } + + /** @return the cached map file */ + public File getFile() { + return file; + } + + /** @return which service supplied the map */ + public DensityMapSource getSource() { + return source; + } + + /** @return the file format */ + public DensityFileFormat getFormat() { + return format; + } + + /** @return what the map values mean; never {@link DensityMapKind#AUTO} */ + public DensityMapKind getKind() { + return kind; + } + + /** @return the PDB entry, or null */ + public PdbId getPdbId() { + return pdbId; + } + + /** @return the EMDB entry, or null */ + public String getEmdbId() { + return emdbId; + } + + /** @return the URL the map was fetched from */ + public String getSourceUrl() { + return sourceUrl; + } + + /** @return true if the file was already cached rather than downloaded now */ + public boolean isFromCache() { + return fromCache; + } + + /** + * Whether a viewer can contour this file as it stands. + * + * @return false for map coefficients, which need a Fourier + * transform first + * @see DensityFileFormat#isJmolLoadable() + */ + public boolean isRenderable() { + return format != null && format.isJmolLoadable(); + } + + /** + * The contour level recommended by the depositors, in absolute map units. + * Normally only available for EMDB maps, where it is the conventional way to + * contour rather than a multiple of sigma. + * + * @return the level, or null if unknown + */ + public Double getRecommendedContourLevel() { + return recommendedContourLevel; + } + + /** + * @return the RMS deviation of the map values, or null if unknown + */ + public Double getSigma() { + return sigma; + } + + /** + * The recommended contour expressed in multiples of sigma, for viewers that + * prefer to work that way. + * + * @return the level divided by sigma, or null if either is unknown + * or sigma is zero + */ + public Double getContourInSigma() { + if (recommendedContourLevel == null || sigma == null || sigma == 0.0) { + return null; + } + return recommendedContourLevel / sigma; + } + + /** @return the size of the cached file in bytes, or 0 if it is missing */ + public long getFileSizeBytes() { + return file == null ? 0 : file.length(); + } + + /** + * The metadata sidecar file for a cached map. + * + * @param mapFile the cached map + * @return the sidecar path, which need not exist + */ + public static File metaFileFor(File mapFile) { + return new File(mapFile.getAbsoluteFile().getParentFile(), mapFile.getName() + META_EXT); + } + + /** + * Writes this result's metadata beside the cached file, so that it can be + * reconstructed later without any network access. + */ + public void writeMeta() { + Properties p = new Properties(); + put(p, "source", source); + put(p, "format", format); + put(p, "kind", kind); + put(p, "pdbId", pdbId == null ? null : pdbId.getId()); + put(p, "emdbId", emdbId); + put(p, "url", sourceUrl); + put(p, "downloaded", Instant.now().toString()); + put(p, "bytes", getFileSizeBytes()); + put(p, "contourLevel", recommendedContourLevel); + put(p, "sigma", sigma); + File meta = metaFileFor(file); + try (OutputStream out = Files.newOutputStream(meta.toPath())) { + p.store(out, "BioJava density map metadata"); + } catch (IOException e) { + // Losing the sidecar costs us the offline description, not the map itself. + logger.warn("Could not write density metadata [{}]: {}", meta, e.getMessage()); + } + } + + /** + * Reconstructs a result from a cached file and its metadata sidecar. + * + * @param mapFile the cached map + * @return the reconstructed result, or null if the sidecar is + * missing or unusable + */ + public static DensityMapResult readMeta(File mapFile) { + File meta = metaFileFor(mapFile); + if (!meta.isFile()) { + return null; + } + Properties p = new Properties(); + try (InputStream in = Files.newInputStream(meta.toPath())) { + p.load(in); + } catch (IOException e) { + logger.warn("Could not read density metadata [{}]: {}", meta, e.getMessage()); + return null; + } + try { + String pdb = p.getProperty("pdbId"); + return new DensityMapResult(mapFile, + DensityMapSource.valueOf(p.getProperty("source")), + DensityFileFormat.valueOf(p.getProperty("format")), + DensityMapKind.valueOf(p.getProperty("kind")), + pdb == null || pdb.isEmpty() ? null : new PdbId(pdb), + emptyToNull(p.getProperty("emdbId")), + p.getProperty("url"), + true, + parseDouble(p.getProperty("contourLevel")), + parseDouble(p.getProperty("sigma"))); + } catch (RuntimeException e) { + logger.warn("Density metadata [{}] is unusable: {}", meta, e.getMessage()); + return null; + } + } + + private static void put(Properties p, String key, Object value) { + p.setProperty(key, value == null ? "" : String.valueOf(value)); + } + + private static String emptyToNull(String s) { + return s == null || s.isEmpty() ? null : s; + } + + private static Double parseDouble(String s) { + if (s == null || s.isEmpty()) { + return null; + } + try { + return Double.valueOf(s); + } catch (NumberFormatException e) { + return null; + } + } + + @Override + public String toString() { + return String.format("%s %s map from %s (%s, %d bytes)%s", + pdbId == null ? emdbId : pdbId.getId(), kind, source, format, getFileSizeBytes(), + isRenderable() ? "" : " [needs an FFT before it can be displayed]"); + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapSource.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapSource.java new file mode 100644 index 0000000000..a0f2e1a365 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapSource.java @@ -0,0 +1,84 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +/** + * A remote service that density data can be fetched from. + *

+ * These differ enormously in size for the same entry, which is why more than one + * is supported. For PDB entry 1cbs a density-server slice at the coarsest detail + * level is about 210 kB and contains both the 2Fo-Fc and Fo-Fc maps, where + * the two pre-computed CCP4 files come to about 2.1 MB together. For cryo-EM + * the gap is far wider: the primary map of EMD-0262 is about 106 MB, against + * roughly 480 kB for the equivalent density-server slice. + *

+ * Note that RCSB's own edmaps.rcsb.org service, which used to serve + * DSN6 and MTZ files, was shut down in October 2024. The map coefficients + * published with the wwPDB validation reports replaced it, but those are + * structure factors rather than a sampled grid; see + * {@link DensityFileFormat#MAP_COEFFICIENTS_CIF_GZ}. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public enum DensityMapSource { + + /** + * RCSB's Mol* density server at maps.rcsb.org, which serves + * downsampled BinaryCIF volume slices for both X-ray and EM entries. + */ + RCSB_VOLUME_SERVER("rcsbvs"), + + /** + * PDBe's Mol* density server, equivalent to {@link #RCSB_VOLUME_SERVER}. + */ + PDBE_VOLUME_SERVER("pdbevs"), + + /** + * PDBe's pre-computed full CCP4 maps. This is the source Jmol itself uses for + * its built-in map-loading shortcuts. + */ + PDBE_CCP4("pdbe"), + + /** + * The primary map of an EMDB entry, at full resolution. Can be very large. + */ + EMDB_MAP("emdb"), + + /** + * Map coefficients from the wwPDB validation reports. Archival only: these + * cannot be displayed without an FFT. + */ + WWPDB_MAP_COEFFICIENTS("wwpdb"); + + private final String fileToken; + + DensityMapSource(String fileToken) { + this.fileToken = fileToken; + } + + /** + * The short token used to distinguish this source in a cached file name, so + * that maps of the same kind from different sources never collide. + * + * @return the token + */ + public String getFileToken() { + return fileToken; + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapTooLargeException.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapTooLargeException.java new file mode 100644 index 0000000000..ce62e36d1d --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/DensityMapTooLargeException.java @@ -0,0 +1,84 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.IOException; + +/** + * Thrown when a density map exceeds the configured download size limit. + *

+ * Cryo-EM primary maps in particular can be very large — hundreds of + * megabytes is common and gigabyte maps exist — while a downsampled slice + * of the same map from a density server is usually a fraction of a percent of + * that size and quite adequate for display. Rather than failing, the fallback + * chain treats this as "try the next source", which is exactly why the density + * servers are tried ahead of the full-resolution archives. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class DensityMapTooLargeException extends IOException { + + private static final long serialVersionUID = 1L; + + private final long sizeBytes; + private final long limitBytes; + + /** + * @param url the resource that was too large + * @param sizeBytes its size, or a negative value if only a lower bound is known + * @param limitBytes the configured limit + */ + public DensityMapTooLargeException(String url, long sizeBytes, long limitBytes) { + super(String.format("%s is %s, which exceeds the %s download limit.", + url, describe(sizeBytes), describe(limitBytes))); + this.sizeBytes = sizeBytes; + this.limitBytes = limitBytes; + } + + private static String describe(long bytes) { + if (bytes < 0) { + return "of unknown size"; + } + if (bytes < 1024) { + return bytes + " B"; + } + double value = bytes; + String[] units = {"kB", "MB", "GB", "TB"}; + int unit = -1; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit++; + } + return String.format("%.1f %s", value, units[unit]); + } + + /** + * @return the size of the resource in bytes, or a negative value if unknown + */ + public long getSizeBytes() { + return sizeBytes; + } + + /** + * @return the configured limit in bytes + */ + public long getLimitBytes() { + return limitBytes; + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryInfo.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryInfo.java new file mode 100644 index 0000000000..7c8b0dd302 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryInfo.java @@ -0,0 +1,85 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +/** + * The few facts about an EMDB entry's primary map that matter when fetching and + * displaying it. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class EmdbEntryInfo { + + private final String emdbId; + private final Double recommendedContourLevel; + private final Double sigma; + private final Long mapSizeBytes; + + /** + * @param emdbId the entry identifier, canonical form + * @param recommendedContourLevel the author-recommended contour level in absolute + * map units, or null + * @param sigma the RMS deviation of the map values, or null + * @param mapSizeBytes the size of the primary map file, or null + */ + public EmdbEntryInfo(String emdbId, Double recommendedContourLevel, Double sigma, Long mapSizeBytes) { + this.emdbId = emdbId; + this.recommendedContourLevel = recommendedContourLevel; + this.sigma = sigma; + this.mapSizeBytes = mapSizeBytes; + } + + /** @return the entry identifier in canonical EMD-1234 form */ + public String getEmdbId() { + return emdbId; + } + + /** + * The contour level the depositors recommend, in absolute map units. EM maps + * are conventionally displayed at this level rather than at a multiple of + * sigma, so it is the right default for a viewer. + * + * @return the level, or null if the entry does not state one + */ + public Double getRecommendedContourLevel() { + return recommendedContourLevel; + } + + /** + * @return the RMS deviation of the map values, which converts between absolute + * and sigma-relative contour levels, or null if unknown + */ + public Double getSigma() { + return sigma; + } + + /** + * @return the size of the full primary map in bytes, or null if + * unknown. Used to decline a download before it starts. + */ + public Long getMapSizeBytes() { + return mapSizeBytes; + } + + @Override + public String toString() { + return String.format("%s[contour=%s, sigma=%s, %s bytes]", + emdbId, recommendedContourLevel, sigma, mapSizeBytes); + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryResolver.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryResolver.java new file mode 100644 index 0000000000..fab0101b34 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbEntryResolver.java @@ -0,0 +1,382 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.align.util.URLConnectionTools; +import org.biojava.nbio.structure.io.LocalPDBDirectory.FetchBehavior; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Finds the EMDB entry, if any, associated with a PDB entry, and reads the few + * facts about its map that are needed to fetch and display it. + *

+ * The primary lookup is EMDB's own search API, which returns the identifier and + * the author-recommended contour level together in a single small CSV response. + * That is also the route Jmol uses, so BioJava's answer matches what a user sees + * elsewhere. If it fails, RCSB's entry API is consulted for the identifier alone. + *

+ * The experimental method is deliberately not inferred from a structure's + * resolution: BioJava is known to mis-parse resolution for some cryo-EM entries + * (biojava/biojava#1000), so the presence of an EMDB identifier is the reliable + * signal. + *

+ * Answers are cached on disk. Under + * {@link FetchBehavior#LOCAL_ONLY} the cached answer is used whatever its age and + * no connection is opened. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class EmdbEntryResolver { + + private static final Logger logger = LoggerFactory.getLogger(EmdbEntryResolver.class); + + /** + * Default template for the EMDB search that maps a PDB entry to the EM + * reconstructions it was fitted into, returning the contour level as well. + */ + public static final String DEFAULT_SEARCH_URL_TEMPLATE = + "https://www.ebi.ac.uk/emdb/api/search/fitted_pdbs:{pdbid_lc}?fl=emdb_id,map_contour_level_value&wt=csv"; + + /** Default template for the EMDB entry map metadata. */ + public static final String DEFAULT_MAP_INFO_URL_TEMPLATE = + "https://www.ebi.ac.uk/emdb/api/entry/map/{emdb_id}"; + + /** Default template for the RCSB entry record, used as a fallback. */ + public static final String DEFAULT_RCSB_ENTRY_URL_TEMPLATE = + "https://data.rcsb.org/rest/v1/core/entry/{pdbid_lc}"; + + private static final int TIMEOUT_MILLIS = 30000; + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static String searchUrlTemplate = DEFAULT_SEARCH_URL_TEMPLATE; + private static String mapInfoUrlTemplate = DEFAULT_MAP_INFO_URL_TEMPLATE; + private static String rcsbEntryUrlTemplate = DEFAULT_RCSB_ENTRY_URL_TEMPLATE; + + private File cacheRoot; + private FetchBehavior fetchBehavior = FetchBehavior.FETCH_FILES; + private int mappingMaxAgeDays = 30; + + /** + * @param cacheRoot the BioJava cache directory + */ + public EmdbEntryResolver(File cacheRoot) { + this.cacheRoot = cacheRoot; + } + + /** @param cacheRoot the BioJava cache directory */ + public void setCacheRoot(File cacheRoot) { + this.cacheRoot = cacheRoot; + } + + /** @param fetchBehavior how aggressively to refresh cached answers */ + public void setFetchBehavior(FetchBehavior fetchBehavior) { + this.fetchBehavior = fetchBehavior == null ? FetchBehavior.FETCH_FILES : fetchBehavior; + } + + /** + * @param days how long a cached mapping stays fresh. Mappings change only when + * an entry is re-released, so this can be generous. + */ + public void setMappingMaxAgeDays(int days) { + this.mappingMaxAgeDays = days; + } + + /** @param template the EMDB search URL template */ + public static void setSearchUrlTemplate(String template) { + searchUrlTemplate = template == null ? DEFAULT_SEARCH_URL_TEMPLATE : template; + } + + /** @param template the EMDB map metadata URL template */ + public static void setMapInfoUrlTemplate(String template) { + mapInfoUrlTemplate = template == null ? DEFAULT_MAP_INFO_URL_TEMPLATE : template; + } + + /** @param template the RCSB entry URL template used as a fallback */ + public static void setRcsbEntryUrlTemplate(String template) { + rcsbEntryUrlTemplate = template == null ? DEFAULT_RCSB_ENTRY_URL_TEMPLATE : template; + } + + /** Restores all default URL templates. */ + public static void resetToDefaults() { + searchUrlTemplate = DEFAULT_SEARCH_URL_TEMPLATE; + mapInfoUrlTemplate = DEFAULT_MAP_INFO_URL_TEMPLATE; + rcsbEntryUrlTemplate = DEFAULT_RCSB_ENTRY_URL_TEMPLATE; + } + + /** + * The EMDB entries a PDB entry was fitted into. + * + * @param pdbId the PDB entry + * @return the EMDB identifiers, most relevant first; empty if the entry is not + * an EM structure or has no associated map + */ + public List getEmdbIds(PdbId pdbId) { + Mapping cached = readMapping(pdbId); + if (cached != null && (fetchBehavior == FetchBehavior.LOCAL_ONLY || cached.isFresh(mappingMaxAgeDays))) { + return cached.emdbIds; + } + if (fetchBehavior == FetchBehavior.LOCAL_ONLY) { + return Collections.emptyList(); + } + + Mapping fetched = searchEmdb(pdbId); + if (fetched == null) { + fetched = queryRcsb(pdbId); + } + if (fetched == null) { + // Serve a stale answer rather than nothing when the services are unreachable. + return cached == null ? Collections.emptyList() : cached.emdbIds; + } + writeMapping(pdbId, fetched); + return fetched.emdbIds; + } + + /** + * The contour level the depositors recommend for the map a PDB entry was fitted + * into, as reported by the EMDB search. + * + * @param pdbId the PDB entry + * @return the level in absolute map units, or null if unknown + */ + public Double getContourLevelFor(PdbId pdbId) { + Mapping cached = readMapping(pdbId); + if (cached != null && (fetchBehavior == FetchBehavior.LOCAL_ONLY || cached.isFresh(mappingMaxAgeDays))) { + return cached.contourLevel; + } + getEmdbIds(pdbId); + Mapping refreshed = readMapping(pdbId); + return refreshed == null ? null : refreshed.contourLevel; + } + + /** + * Reads an EMDB entry's map metadata: contour level, RMS deviation and the size + * of the primary map file. + *

+ * The response is cached verbatim, so asking twice costs one request. + * + * @param emdbId the EMDB entry, in any accepted form + * @return the metadata, or null if it could not be obtained + */ + public EmdbEntryInfo getEntryInfo(String emdbId) { + String canonical = DensityMapRequest.normalizeEmdbId(emdbId); + File cacheFile = DensityCacheLayout.emdbMapInfoFile(cacheRoot, canonical); + + String json = null; + if (cacheFile.isFile()) { + try { + json = new String(Files.readAllBytes(cacheFile.toPath()), StandardCharsets.UTF_8); + } catch (IOException e) { + logger.debug("Could not read cached EMDB metadata [{}]: {}", cacheFile, e.getMessage()); + } + } + if (json == null) { + if (fetchBehavior == FetchBehavior.LOCAL_ONLY) { + return null; + } + String url = UrlTemplates.expand(mapInfoUrlTemplate, UrlTemplates.values(null, canonical, -1)); + try { + json = read(new URL(url)); + } catch (IOException e) { + logger.warn("Could not read EMDB metadata for {}: {}", canonical, e.getMessage()); + return null; + } + writeCache(cacheFile, json); + } + + try { + JsonNode map = MAPPER.readTree(json).path("map"); + Double contour = null; + JsonNode contours = map.path("contour_list").path("contour"); + for (JsonNode c : contours) { + if (contour == null || c.path("primary").asBoolean(false)) { + contour = c.path("level").isNumber() ? c.path("level").asDouble() : contour; + } + } + Double sigma = map.path("statistics").path("std").isNumber() + ? map.path("statistics").path("std").asDouble() : null; + Long bytes = map.path("size_kbytes").isNumber() + ? map.path("size_kbytes").asLong() * 1024L : null; + return new EmdbEntryInfo(canonical, contour, sigma, bytes); + } catch (IOException | RuntimeException e) { + logger.warn("Could not parse EMDB metadata for {}: {}", canonical, e.getMessage()); + return null; + } + } + + private Mapping searchEmdb(PdbId pdbId) { + String url = UrlTemplates.expand(searchUrlTemplate, + UrlTemplates.values(DensityCacheLayout.shortIdOrFull(pdbId), null, -1)); + try { + String csv = read(new URL(url)); + List ids = new ArrayList<>(); + Double contour = null; + String[] lines = csv.split("\\R"); + for (int i = 1; i < lines.length; i++) { // line 0 is the header + String line = lines[i].trim(); + if (line.isEmpty()) { + continue; + } + String[] cols = line.split(","); + if (cols.length > 0 && !cols[0].isEmpty()) { + ids.add(DensityMapRequest.normalizeEmdbId(cols[0])); + } + if (contour == null && cols.length > 1 && !cols[1].isEmpty()) { + try { + contour = Double.valueOf(cols[1].trim()); + } catch (NumberFormatException ignored) { + // the column is optional and occasionally blank + } + } + } + return new Mapping(ids, contour, Instant.now()); + } catch (IOException e) { + logger.debug("EMDB search for {} failed: {}", pdbId.getId(), e.getMessage()); + return null; + } + } + + private Mapping queryRcsb(PdbId pdbId) { + String url = UrlTemplates.expand(rcsbEntryUrlTemplate, + UrlTemplates.values(DensityCacheLayout.shortIdOrFull(pdbId), null, -1)); + try { + JsonNode root = MAPPER.readTree(read(new URL(url))); + JsonNode ids = root.path("rcsb_entry_container_identifiers").path("emdb_ids"); + List result = new ArrayList<>(); + for (JsonNode id : ids) { + result.add(DensityMapRequest.normalizeEmdbId(id.asText())); + } + return new Mapping(result, null, Instant.now()); + } catch (IOException | RuntimeException e) { + logger.debug("RCSB entry lookup for {} failed: {}", pdbId.getId(), e.getMessage()); + return null; + } + } + + private String read(URL url) throws IOException { + try (InputStream in = URLConnectionTools.getInputStream(url, true, TIMEOUT_MILLIS); + BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + StringBuilder sb = new StringBuilder(); + char[] buffer = new char[8192]; + int n; + while ((n = reader.read(buffer)) != -1) { + sb.append(buffer, 0, n); + } + return sb.toString(); + } + } + + private Mapping readMapping(PdbId pdbId) { + File file = DensityCacheLayout.emdbMappingFile(cacheRoot, pdbId); + if (!file.isFile()) { + return null; + } + Properties p = new Properties(); + try (InputStream in = Files.newInputStream(file.toPath())) { + p.load(in); + } catch (IOException e) { + return null; + } + String ids = p.getProperty("emdbIds", ""); + List list = ids.isEmpty() ? Collections.emptyList() : Arrays.asList(ids.split(",")); + Double contour = null; + try { + String c = p.getProperty("contourLevel", ""); + contour = c.isEmpty() ? null : Double.valueOf(c); + } catch (NumberFormatException ignored) { + // leave it null + } + Instant retrieved; + try { + retrieved = Instant.parse(p.getProperty("retrieved")); + } catch (RuntimeException e) { + retrieved = Instant.EPOCH; + } + return new Mapping(list, contour, retrieved); + } + + private void writeMapping(PdbId pdbId, Mapping mapping) { + File file = DensityCacheLayout.emdbMappingFile(cacheRoot, pdbId); + File dir = file.getParentFile(); + if (!dir.isDirectory() && !dir.mkdirs()) { + logger.debug("Could not create [{}]", dir); + return; + } + Properties p = new Properties(); + p.setProperty("pdbId", DensityCacheLayout.shortIdOrFull(pdbId)); + p.setProperty("emdbIds", String.join(",", mapping.emdbIds)); + p.setProperty("contourLevel", mapping.contourLevel == null ? "" : mapping.contourLevel.toString()); + p.setProperty("retrieved", mapping.retrieved.toString()); + try (OutputStream out = Files.newOutputStream(file.toPath())) { + p.store(out, "BioJava PDB to EMDB mapping"); + } catch (IOException e) { + logger.debug("Could not cache the EMDB mapping for {}: {}", pdbId.getId(), e.getMessage()); + } + } + + private void writeCache(File file, String content) { + File dir = file.getParentFile(); + if (!dir.isDirectory() && !dir.mkdirs()) { + return; + } + try { + Files.write(file.toPath(), content.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + logger.debug("Could not cache [{}]: {}", file, e.getMessage()); + } + } + + /** A cached PDB-to-EMDB answer. */ + private static final class Mapping { + final List emdbIds; + final Double contourLevel; + final Instant retrieved; + + Mapping(List emdbIds, Double contourLevel, Instant retrieved) { + this.emdbIds = Collections.unmodifiableList(new ArrayList<>(emdbIds)); + this.contourLevel = contourLevel; + this.retrieved = retrieved; + } + + boolean isFresh(int maxAgeDays) { + return Duration.between(retrieved, Instant.now()).toDays() < maxAgeDays; + } + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbMapProvider.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbMapProvider.java new file mode 100644 index 0000000000..7f90814c30 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/EmdbMapProvider.java @@ -0,0 +1,142 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.File; +import java.io.IOException; +import java.net.URL; + +/** + * Fetches the full-resolution primary map of an EMDB entry. + *

+ * These are gzipped CCP4/MRC files, and they are big: the primary map of + * EMD-0262 is about 106 MB, and gigabyte maps exist. For most display + * purposes a slice from a density server is a far better trade — a few + * hundred kilobytes for the same entry — which is why + * {@link DensityMapCache} tries {@link VolumeServerProvider} first and only falls + * back here. Use this source when the full sampling genuinely matters. + *

+ * The size limit is checked against the size EMDB itself reports before any of + * the body is transferred, so exceeding it costs one small metadata request + * rather than a partial download. + *

+ * Jmol recognises gzip from the file's magic bytes, so the cached + * .map.gz can be handed to it without decompressing first. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class EmdbMapProvider extends AbstractDensityMapProvider { + + /** Default base URL of the EMDB archive. */ + public static final String DEFAULT_SERVER_URL = "https://ftp.ebi.ac.uk/pub/databases/emdb/structures/"; + + /** Default path template for an entry's primary map. */ + public static final String DEFAULT_MAP_TEMPLATE = "{emdb_id}/map/emd_{emdb_num}.map.gz"; + + private static String serverBaseUrl = DEFAULT_SERVER_URL; + private static String mapTemplate = DEFAULT_MAP_TEMPLATE; + + private final EmdbEntryResolver resolver; + + /** + * @param cacheRoot the BioJava cache directory + * @param resolver used to find the EMDB entry for a PDB entry and to read its + * contour level and size + */ + public EmdbMapProvider(File cacheRoot, EmdbEntryResolver resolver) { + super(cacheRoot); + this.resolver = resolver; + } + + /** @return the base URL of the EMDB archive */ + public static String getServerBaseUrl() { + return serverBaseUrl; + } + + /** @param url the base URL; a trailing slash is added if missing */ + public static void setServerBaseUrl(String url) { + serverBaseUrl = url == null ? DEFAULT_SERVER_URL : (url.endsWith("/") ? url : url + "/"); + } + + /** @param template the path template for an entry's primary map */ + public static void setMapUrlTemplate(String template) { + mapTemplate = template == null ? DEFAULT_MAP_TEMPLATE : template; + } + + /** Restores the default server and template. */ + public static void resetToDefaults() { + serverBaseUrl = DEFAULT_SERVER_URL; + mapTemplate = DEFAULT_MAP_TEMPLATE; + } + + @Override + public DensityMapSource getSource() { + return DensityMapSource.EMDB_MAP; + } + + @Override + public DensityFileFormat getFormat() { + return DensityFileFormat.CCP4_GZ; + } + + @Override + public boolean supports(DensityMapKind kind) { + return kind == DensityMapKind.EM; + } + + /** + * Builds the URL of an entry's primary map without fetching it. + * + * @param emdbId the EMDB entry, in any accepted form + * @return the URL as a string + */ + public String buildUrl(String emdbId) { + return serverBaseUrl + UrlTemplates.expand(mapTemplate, UrlTemplates.values(null, emdbId, -1)); + } + + @Override + public DensityMapResult fetch(DensityMapRequest request) throws IOException { + if (!supports(request.getKind())) { + return null; + } + String emdbId = request.getEmdbId(); + if (emdbId == null) { + return null; + } + + EmdbEntryInfo info = resolver == null ? null : resolver.getEntryInfo(emdbId); + + // Decline before transferring anything: EMDB reports the map size in its + // metadata, so an oversized map costs one small request rather than a + // partial multi-hundred-megabyte download. + long limit = effectiveMaxBytes(request); + if (limit > 0 && info != null && info.getMapSizeBytes() != null && info.getMapSizeBytes() > limit) { + URL url = new URL(buildUrl(emdbId)); + reportTooLarge(url, info.getMapSizeBytes(), limit, request); + throw new DensityMapTooLargeException(url.toString(), info.getMapSizeBytes(), limit); + } + + URL url = new URL(buildUrl(emdbId)); + File target = DensityCacheLayout.emdbMapFile(effectiveCacheRoot(request), emdbId, + getSource(), getFormat(), null); + return obtain(request, url, target, DensityMapKind.EM, emdbId, + info == null ? null : info.getRecommendedContourLevel(), + info == null ? null : info.getSigma()); + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/NoDensityMapException.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/NoDensityMapException.java new file mode 100644 index 0000000000..6691b584d9 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/NoDensityMapException.java @@ -0,0 +1,99 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.biojava.nbio.structure.PdbId; + +/** + * Thrown when no enabled source could supply a density map for an entry. + *

+ * This is a routine outcome rather than an error: not every entry has density. + * Structures deposited without structure factors have none at all (4HHB, for + * instance), and a cryo-EM entry has no X-ray maps by construction. The + * per-source reasons are kept so that a caller — particularly a user + * interface — can explain why rather than just reporting a failure. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class NoDensityMapException extends IOException { + + private static final long serialVersionUID = 1L; + + private final PdbId pdbId; + private final DensityMapKind kind; + private final Map attempts; + + /** + * @param pdbId the entry that was requested, may be null for a + * request made by EMDB identifier + * @param kind the kind of map that was requested + * @param attempts what happened at each source that was tried, in the order tried + */ + public NoDensityMapException(PdbId pdbId, DensityMapKind kind, Map attempts) { + super(buildMessage(pdbId, kind, attempts)); + this.pdbId = pdbId; + this.kind = kind; + this.attempts = attempts == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new LinkedHashMap<>(attempts)); + } + + private static String buildMessage(PdbId pdbId, DensityMapKind kind, Map attempts) { + StringBuilder sb = new StringBuilder("No "); + sb.append(kind == null ? "density" : kind).append(" map is available for "); + sb.append(pdbId == null ? "the requested entry" : pdbId.getId()).append('.'); + if (attempts != null && !attempts.isEmpty()) { + sb.append(" Tried:"); + for (Map.Entry e : attempts.entrySet()) { + sb.append(' ').append(e.getKey()).append(" (").append(e.getValue()).append(");"); + } + sb.setLength(sb.length() - 1); + } + return sb.toString(); + } + + /** + * @return the entry that was requested, or null + */ + public PdbId getPdbId() { + return pdbId; + } + + /** + * @return the kind of map that was requested + */ + public DensityMapKind getKind() { + return kind; + } + + /** + * What happened at each source, in the order they were tried. Suitable for + * building a human-readable explanation. + * + * @return an unmodifiable map from source to reason + */ + public Map getAttempts() { + return attempts; + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/PdbeCcp4MapProvider.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/PdbeCcp4MapProvider.java new file mode 100644 index 0000000000..a825ccfa70 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/PdbeCcp4MapProvider.java @@ -0,0 +1,142 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.File; +import java.io.IOException; +import java.net.URL; + +import org.biojava.nbio.structure.PdbId; + +/** + * Fetches PDBe's pre-computed CCP4 maps. + *

+ * These are full-resolution sampled grids, one file per map kind, and are the + * source Jmol itself uses for its built-in map shortcuts. They are larger than a + * density-server slice — about 1 MB per map for a small entry such as + * 1cbs — but need no interpretation beyond contouring. + *

+ * The service accepts only the lower-case four-character spelling of an + * identifier; 1CBS.ccp4 and pdb_00001cbs.ccp4 both + * return HTTP 404. Entries deposited without structure factors, and cryo-EM + * entries, have no maps here at all. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class PdbeCcp4MapProvider extends AbstractDensityMapProvider { + + /** Default base URL of the PDBe map service. */ + public static final String DEFAULT_SERVER_URL = "https://www.ebi.ac.uk/pdbe/coordinates/files/"; + + /** + * An equivalent base URL serving the same files, kept in the documentation as a + * ready alternative should the primary one change. + */ + public static final String ALTERNATIVE_SERVER_URL = "https://www.ebi.ac.uk/pdbe/entry-files/"; + + /** Default path template for the 2mFo-DFc map. */ + public static final String DEFAULT_TWO_FO_FC_TEMPLATE = "{pdbid_lc}.ccp4"; + + /** Default path template for the mFo-DFc difference map. */ + public static final String DEFAULT_FO_FC_TEMPLATE = "{pdbid_lc}_diff.ccp4"; + + private static String serverBaseUrl = DEFAULT_SERVER_URL; + private static String twoFoFcTemplate = DEFAULT_TWO_FO_FC_TEMPLATE; + private static String foFcTemplate = DEFAULT_FO_FC_TEMPLATE; + + /** + * @param cacheRoot the BioJava cache directory + */ + public PdbeCcp4MapProvider(File cacheRoot) { + super(cacheRoot); + } + + /** @return the base URL of the map service */ + public static String getServerBaseUrl() { + return serverBaseUrl; + } + + /** + * @param url the base URL of the map service; a trailing slash is added if missing + */ + public static void setServerBaseUrl(String url) { + serverBaseUrl = url == null ? DEFAULT_SERVER_URL : (url.endsWith("/") ? url : url + "/"); + } + + /** + * Overrides the path template for a map kind. + * + * @param kind {@link DensityMapKind#TWO_FO_FC} or {@link DensityMapKind#FO_FC} + * @param template a template understood by {@link UrlTemplates} + */ + public static void setPathUrlTemplate(DensityMapKind kind, String template) { + if (kind == DensityMapKind.TWO_FO_FC) { + twoFoFcTemplate = template == null ? DEFAULT_TWO_FO_FC_TEMPLATE : template; + } else if (kind == DensityMapKind.FO_FC) { + foFcTemplate = template == null ? DEFAULT_FO_FC_TEMPLATE : template; + } else { + throw new IllegalArgumentException("PDBe CCP4 maps exist only for 2Fo-Fc and Fo-Fc, not " + kind); + } + } + + /** Restores the default server and templates. */ + public static void resetToDefaults() { + serverBaseUrl = DEFAULT_SERVER_URL; + twoFoFcTemplate = DEFAULT_TWO_FO_FC_TEMPLATE; + foFcTemplate = DEFAULT_FO_FC_TEMPLATE; + } + + @Override + public DensityMapSource getSource() { + return DensityMapSource.PDBE_CCP4; + } + + @Override + public DensityFileFormat getFormat() { + return DensityFileFormat.CCP4; + } + + @Override + public boolean supports(DensityMapKind kind) { + return kind == DensityMapKind.TWO_FO_FC || kind == DensityMapKind.FO_FC; + } + + /** + * Builds the URL for a map without fetching it. + * + * @param pdbId the entry + * @param kind the kind of map + * @return the URL as a string + */ + public String buildUrl(PdbId pdbId, DensityMapKind kind) { + String template = kind == DensityMapKind.FO_FC ? foFcTemplate : twoFoFcTemplate; + return serverBaseUrl + UrlTemplates.expand(template, UrlTemplates.values(urlId(pdbId), null, -1)); + } + + @Override + public DensityMapResult fetch(DensityMapRequest request) throws IOException { + if (request.getPdbId() == null || !supports(request.getKind())) { + return null; + } + URL url = new URL(buildUrl(request.getPdbId(), request.getKind())); + File target = DensityCacheLayout.pdbMapFile(effectiveCacheRoot(request), request.getPdbId(), + request.getKind(), getSource(), getFormat(), null); + return obtain(request, url, target, request.getKind(), null, null, null); + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/UrlTemplates.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/UrlTemplates.java new file mode 100644 index 0000000000..0518f1d4a4 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/UrlTemplates.java @@ -0,0 +1,152 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.StructureException; + +/** + * Expands the named placeholders used in the configurable density-server URL + * templates. + *

+ * The recognised placeholders are: + *

+ *
{pdbid}
the PDB identifier as given
+ *
{pdbid_lc}
the PDB identifier in lower case
+ *
{pdbid_uc}
the PDB identifier in upper case
+ *
{mid}
the two-character divided-archive directory, + * e.g. cb for 1cbs
+ *
{extid}
the extended identifier in lower case, the + * spelling the archive itself uses, e.g. pdb_00001cbs for + * 1cbs
+ *
{emdb_id}
the EMDB identifier, e.g. EMD-0262
+ *
{emdb_num}
the EMDB number alone, e.g. 0262
+ *
{detail}
the density-server detail level
+ *
+ * A placeholder with no supplied value is left in place rather than replaced by + * an empty string, so that a misconfigured template produces an obviously wrong + * URL instead of a subtly wrong one. + *

+ * {mid} and {extid} exist for mirrors rather than for + * the default URLs. The services BioJava fetches from resolve an entry by name + * alone, but a site pointing this code at its own copy of the archive has to + * spell out a directory path — and so does any mirror that publishes one, + * such as EBI. Both the present divided layout and the per-entry layout that + * replaces it in July 2027 are directory paths, and between them the two + * placeholders express either without a code change: + *

+ * {mid}/{pdbid_lc}/{pdbid_lc}_validation_2fo-fc_map_coef.cif.gz
+ * entries/{mid}/{extid}/validation_reports/{extid}_validation_2fo-fc_map_coef.cif.gz
+ * 
+ *

+ * This is deliberately separate from the template mechanism in + * {@code DownloadChemCompProvider}, which resolves a single chemical-component + * identifier with optional substring indices. The two solve different problems: + * here several distinct values are substituted by name. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class UrlTemplates { + + private static final Pattern PLACEHOLDER = Pattern.compile("\\{([a-zA-Z_]+)\\}"); + + private UrlTemplates() { + } + + /** + * Expands a template against a set of named values. + * + * @param template the template string + * @param values the values, keyed by placeholder name without the braces + * @return the expanded string + */ + public static String expand(String template, Map values) { + if (template == null) { + return null; + } + Matcher m = PLACEHOLDER.matcher(template); + StringBuilder out = new StringBuilder(); + int last = 0; + while (m.find()) { + String value = values.get(m.group(1)); + out.append(template, last, m.start()); + // An unknown placeholder is kept verbatim: a template that was configured + // wrongly should fail loudly rather than quietly produce a plausible URL. + out.append(value == null ? m.group(0) : value); + last = m.end(); + } + out.append(template.substring(last)); + return out.toString(); + } + + /** + * Builds the standard value map for an entry. + * + * @param pdbId the PDB identifier, may be null + * @param emdbId the EMDB identifier in any accepted form, may be null + * @param detail the density-server detail level, or a negative value to omit it + * @return a map suitable for {@link #expand(String, Map)} + */ + public static Map values(String pdbId, String emdbId, int detail) { + Map values = new LinkedHashMap<>(); + if (pdbId != null) { + values.put("pdbid", pdbId); + values.put("pdbid_lc", pdbId.toLowerCase()); + values.put("pdbid_uc", pdbId.toUpperCase()); + if (pdbId.length() >= 3) { + // Correct for both spellings: the hash is counted from the right hand + // end, so 1cbs and pdb_00001cbs both yield "cb". + values.put("mid", org.biojava.nbio.structure.io.LocalPDBDirectory.getMiddleHash(pdbId)); + } + String extendedId = extendedId(pdbId); + if (extendedId != null) { + values.put("extid", extendedId); + } + } + if (emdbId != null) { + values.put("emdb_id", DensityMapRequest.normalizeEmdbId(emdbId)); + values.put("emdb_num", DensityMapRequest.emdbNumber(emdbId)); + } + if (detail >= 0) { + values.put("detail", Integer.toString(detail)); + } + return values; + } + + /** + * The extended spelling of an identifier, in the lower case the archive uses. + * + * @param pdbId an identifier in either spelling + * @return the extended spelling, or null if the argument is + * neither a short nor an extended identifier, in which case + * {extid} is left unexpanded rather than guessed at + */ + private static String extendedId(String pdbId) { + try { + return PdbId.toExtendedId(pdbId).toLowerCase(); + } catch (StructureException e) { + return null; + } + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/VolumeServerProvider.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/VolumeServerProvider.java new file mode 100644 index 0000000000..1700f3a472 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/VolumeServerProvider.java @@ -0,0 +1,246 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +/** + * Fetches downsampled volume slices from a Mol* density server. + *

+ * This is usually the most economical source by a wide margin. For PDB entry + * 1cbs the coarsest slice is about 210 kB against roughly 2.1 MB for + * the two pre-computed CCP4 files, and for cryo-EM the difference is far larger + * still: about 480 kB against the 106 MB primary map of EMD-0262. A + * single response carries both the 2Fo-Fc and Fo-Fc data blocks for X-ray + * entries. + *

+ * Caveats worth knowing. Responses are chunked, with no + * Content-Length, no ETag and no + * Last-Modified, so the usual server-provided validation metadata is + * unavailable; the size recorded for a cached slice is the byte count observed + * during our own download instead. Detail levels are not linear either — + * for a small entry the response stops growing past detail 1, while for a large + * EM map the step from detail 2 to 3 multiplies the size several-fold. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class VolumeServerProvider extends AbstractDensityMapProvider { + + /** + * Which density server to talk to. + */ + public enum Host { + /** RCSB's server at maps.rcsb.org. */ + RCSB(DensityMapSource.RCSB_VOLUME_SERVER, "https://maps.rcsb.org/", 3), + /** PDBe's server at www.ebi.ac.uk/pdbe/volume-server. */ + PDBE(DensityMapSource.PDBE_VOLUME_SERVER, "https://www.ebi.ac.uk/pdbe/volume-server/", 6); + + private final DensityMapSource source; + private final String defaultBaseUrl; + private final int defaultDetail; + + Host(DensityMapSource source, String defaultBaseUrl, int defaultDetail) { + this.source = source; + this.defaultBaseUrl = defaultBaseUrl; + this.defaultDetail = defaultDetail; + } + + /** @return the source constant this host corresponds to */ + public DensityMapSource getSource() { + return source; + } + + /** @return the default base URL */ + public String getDefaultBaseUrl() { + return defaultBaseUrl; + } + + /** @return the detail level this host's own clients use by default */ + public int getDefaultDetail() { + return defaultDetail; + } + } + + /** Default path template for an X-ray entry's whole unit cell. */ + public static final String DEFAULT_XRAY_CELL_TEMPLATE = "x-ray/{pdbid_lc}/cell?detail={detail}&encoding={encoding}"; + + /** Default path template for an EM entry's whole cell. */ + public static final String DEFAULT_EM_CELL_TEMPLATE = "em/emd-{emdb_num}/cell?detail={detail}&encoding={encoding}"; + + private final Host host; + private String baseUrl; + private int detail; + private String encoding = "bcif"; + + /** + * @param cacheRoot the BioJava cache directory + * @param host which density server to use + */ + public VolumeServerProvider(File cacheRoot, Host host) { + super(cacheRoot); + this.host = host; + this.baseUrl = host.getDefaultBaseUrl(); + this.detail = host.getDefaultDetail(); + } + + /** @return which density server this instance uses */ + public Host getHost() { + return host; + } + + /** @return the base URL in use */ + public String getBaseUrl() { + return baseUrl; + } + + /** @param baseUrl the base URL; a trailing slash is added if missing */ + public void setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl == null ? host.getDefaultBaseUrl() + : (baseUrl.endsWith("/") ? baseUrl : baseUrl + "/"); + } + + /** @return the detail level requested from the server */ + public int getDetail() { + return detail; + } + + /** + * Sets the detail level. Higher means a finer grid and a larger download; the + * relationship is neither linear nor the same for every entry. + * + * @param detail the level, normally 0 to 6 + */ + public void setDetail(int detail) { + this.detail = detail; + } + + /** @return the encoding requested, either bcif or cif */ + public String getEncoding() { + return encoding; + } + + /** + * @param encoding bcif for BinaryCIF (default, and much smaller) or + * cif for text + */ + public void setEncoding(String encoding) { + this.encoding = encoding == null ? "bcif" : encoding; + } + + @Override + public DensityMapSource getSource() { + return host.getSource(); + } + + @Override + public DensityFileFormat getFormat() { + return "cif".equalsIgnoreCase(encoding) ? DensityFileFormat.CIF_VOLUME : DensityFileFormat.BCIF_VOLUME; + } + + @Override + public boolean supports(DensityMapKind kind) { + return kind == DensityMapKind.TWO_FO_FC || kind == DensityMapKind.FO_FC || kind == DensityMapKind.EM; + } + + /** + * Builds the URL for a request without fetching it. + * + * @param request the request; its kind must be concrete + * @return the URL as a string, or null if the request cannot be served + */ + public String buildUrl(DensityMapRequest request) { + if (request.getKind() == DensityMapKind.EM) { + if (request.getEmdbId() == null) { + return null; + } + return baseUrl + UrlTemplates.expand(withEncoding(DEFAULT_EM_CELL_TEMPLATE), + UrlTemplates.values(null, request.getEmdbId(), detail)); + } + if (request.getPdbId() == null) { + return null; + } + return baseUrl + UrlTemplates.expand(withEncoding(DEFAULT_XRAY_CELL_TEMPLATE), + UrlTemplates.values(urlId(request.getPdbId()), null, detail)); + } + + private String withEncoding(String template) { + return template.replace("{encoding}", encoding); + } + + @Override + public DensityMapResult fetch(DensityMapRequest request) throws IOException { + if (!supports(request.getKind())) { + return null; + } + String urlString = buildUrl(request); + if (urlString == null) { + return null; + } + URL url = new URL(urlString); + + // The detail level changes the content, so it has to be part of the cache key. + String qualifier = "d" + detail; + File target; + if (request.getKind() == DensityMapKind.EM) { + target = DensityCacheLayout.emdbMapFile(effectiveCacheRoot(request), request.getEmdbId(), + getSource(), getFormat(), qualifier); + } else { + // One response carries both the 2Fo-Fc and the Fo-Fc blocks, so the two + // kinds share a cache entry: asking for each separately would otherwise + // download and store the identical file twice. Which block to read is + // decided at display time, not here. + target = DensityCacheLayout.pdbMapFile(effectiveCacheRoot(request), request.getPdbId(), + DensityCacheLayout.BOTH_KINDS_TOKEN, getSource(), getFormat(), qualifier); + } + DensityMapResult result = obtain(request, url, target, request.getKind(), request.getEmdbId(), null, null); + + if (request.getKind() == DensityMapKind.FO_FC) { + return presentAsDifferenceMap(result); + } + return result; + } + + /** + * Points the result at the companion file name that makes Jmol read the + * difference-map block. See + * {@link DensityCacheLayout#differenceMarkerFile(File)} for why the marker has + * to be in the name. + *

+ * The companion is a hard link where the filesystem allows one, so the second + * name costs no additional space; a copy is only made if linking is refused. + */ + private DensityMapResult presentAsDifferenceMap(DensityMapResult result) throws IOException { + File marker = DensityCacheLayout.differenceMarkerFile(result.getFile()); + if (!marker.isFile() || marker.length() != result.getFile().length()) { + Files.deleteIfExists(marker.toPath()); + try { + Files.createLink(marker.toPath(), result.getFile().toPath()); + } catch (IOException | UnsupportedOperationException e) { + Files.copy(result.getFile().toPath(), marker.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } + return new DensityMapResult(marker, result.getSource(), result.getFormat(), result.getKind(), + result.getPdbId(), result.getEmdbId(), result.getSourceUrl(), result.isFromCache(), + result.getRecommendedContourLevel(), result.getSigma()); + } +} diff --git a/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/WwpdbMapCoefficientsProvider.java b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/WwpdbMapCoefficientsProvider.java new file mode 100644 index 0000000000..bd1b066a64 --- /dev/null +++ b/biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/WwpdbMapCoefficientsProvider.java @@ -0,0 +1,219 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import java.io.File; +import java.io.IOException; +import java.net.URL; + +import org.biojava.nbio.structure.PdbId; + +/** + * Fetches the map coefficients published alongside the wwPDB validation reports. + *

+ * These are not density maps. They are structure-factor amplitudes and + * phases in mmCIF, exactly as used to produce the pictures in a validation + * report, and a Fourier transform is required before anything can be drawn from + * them — gemmi sf2map, or cif2mtz followed by + * CCP4's fft. Nothing in BioJava or Jmol will render them. + *

+ * They are supported because this is the route RCSB documents since + * edmaps.rcsb.org was shut down in October 2024, and because they + * are the authoritative archival form. For anything that needs to be displayed, + * prefer {@link PdbeCcp4MapProvider} or {@link VolumeServerProvider}. Accordingly + * this source is disabled by default in {@link DensityMapCache}. + *

+ * One useful property: these servers return the content MD5 as the HTTP + * ETag, so downloads from here are checksum-verified automatically. + *

+ * The URLs are built against the documented download endpoint, which resolves an + * entry by file name, rather than against the divided archive path. This is the + * only provider here that ever had a choice — the density servers and the + * EMDB archive are addressed by identifier already — and it matters because + * the PDB moves to extended identifiers and a per-entry directory layout in July + * 2027. A name survives that move; a constructed directory path does not. Mirrors + * that publish directories instead of an endpoint are still reachable, through + * {@link #DIVIDED_TWO_FO_FC_TEMPLATE} and {@link #ENTRIES_TWO_FO_FC_TEMPLATE}. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class WwpdbMapCoefficientsProvider extends AbstractDensityMapProvider { + + /** Default base URL, the wwPDB validation report download endpoint. */ + public static final String DEFAULT_SERVER_URL = "https://files.wwpdb.org/validation/download/"; + + /** An RCSB mirror serving byte-identical files. */ + public static final String RCSB_MIRROR_URL = "https://files.rcsb.org/validation/download/"; + + /** + * The wwPDB beta archive, which already holds the re-organised content that + * replaces the current archive on 21 July 2027 and serves the same endpoint. + *

+ * Deliberately not the default, despite being the newer archive. The wwPDB + * describes this host as transitional: on the cutover date the beta archive + * replaces the main one, after which the beta URL is supported by redirection + * for three years. So it is the hostname that needs changing, twice, whereas + * {@link #DEFAULT_SERVER_URL} becomes the new archive and needs changing never. + *

+ * Its value is in testing. Because this host is the post-2027 content today, a + * request against it checks the endpoint against the archive as it will be, + * rather than against the archive as it is. + */ + public static final String BETA_SERVER_URL = "https://files-beta.wwpdb.org/validation/download/"; + + /** + * An EBI mirror serving byte-identical files. + *

+ * Unlike the two above, EBI publishes no name-resolving endpoint — only + * full directory paths — so selecting it means setting the divided + * templates as well: + *

+	 * setServerBaseUrl(EBI_MIRROR_URL);
+	 * setPathUrlTemplate(DensityMapKind.TWO_FO_FC, DIVIDED_TWO_FO_FC_TEMPLATE);
+	 * setPathUrlTemplate(DensityMapKind.FO_FC, DIVIDED_FO_FC_TEMPLATE);
+	 * 
+ * Setting the base alone yields 404s, because the flat file names do not exist + * there. + */ + public static final String EBI_MIRROR_URL = "https://ftp.ebi.ac.uk/pub/databases/pdb/validation_reports/"; + + /** + * Default path template for the 2mFo-DFc coefficients: the file name alone. + *

+ * The endpoint resolves an entry by name, so no directory path is built here. + * That is deliberate. In July 2027 the archive moves to extended identifiers + * and a per-entry directory layout, and a path assembled from a hash and an + * identifier would have to be rewritten for it; a name does not. Both spellings + * of an identifier resolve, so whichever {@code PdbId} yields is accepted. + */ + public static final String DEFAULT_TWO_FO_FC_TEMPLATE = + "{pdbid_lc}_validation_2fo-fc_map_coef.cif.gz"; + + /** Default path template for the mFo-DFc coefficients; see {@link #DEFAULT_TWO_FO_FC_TEMPLATE}. */ + public static final String DEFAULT_FO_FC_TEMPLATE = + "{pdbid_lc}_validation_fo-fc_map_coef.cif.gz"; + + /** + * Path template for the 2mFo-DFc coefficients in the divided archive, for + * mirrors that publish directories rather than an endpoint. + */ + public static final String DIVIDED_TWO_FO_FC_TEMPLATE = + "{mid}/{pdbid_lc}/{pdbid_lc}_validation_2fo-fc_map_coef.cif.gz"; + + /** Path template for the mFo-DFc coefficients in the divided archive. */ + public static final String DIVIDED_FO_FC_TEMPLATE = + "{mid}/{pdbid_lc}/{pdbid_lc}_validation_fo-fc_map_coef.cif.gz"; + + /** + * Path template for the 2mFo-DFc coefficients in the per-entry archive that + * replaces the divided one in July 2027, relative to a base URL ending in + * .../pdb/data/. + *

+ * Provided so that a mirror of the new layout can be used the day it exists, + * without waiting for a release. + */ + public static final String ENTRIES_TWO_FO_FC_TEMPLATE = + "entries/{mid}/{extid}/validation_reports/{extid}_validation_2fo-fc_map_coef.cif.gz"; + + /** Path template for the mFo-DFc coefficients in the per-entry archive. */ + public static final String ENTRIES_FO_FC_TEMPLATE = + "entries/{mid}/{extid}/validation_reports/{extid}_validation_fo-fc_map_coef.cif.gz"; + + private static String serverBaseUrl = DEFAULT_SERVER_URL; + private static String twoFoFcTemplate = DEFAULT_TWO_FO_FC_TEMPLATE; + private static String foFcTemplate = DEFAULT_FO_FC_TEMPLATE; + + /** + * @param cacheRoot the BioJava cache directory + */ + public WwpdbMapCoefficientsProvider(File cacheRoot) { + super(cacheRoot); + } + + /** @return the base URL of the validation report archive */ + public static String getServerBaseUrl() { + return serverBaseUrl; + } + + /** @param url the base URL; a trailing slash is added if missing */ + public static void setServerBaseUrl(String url) { + serverBaseUrl = url == null ? DEFAULT_SERVER_URL : (url.endsWith("/") ? url : url + "/"); + } + + /** + * Overrides the path template for a map kind. + * + * @param kind {@link DensityMapKind#TWO_FO_FC} or {@link DensityMapKind#FO_FC} + * @param template a template understood by {@link UrlTemplates} + */ + public static void setPathUrlTemplate(DensityMapKind kind, String template) { + if (kind == DensityMapKind.TWO_FO_FC) { + twoFoFcTemplate = template == null ? DEFAULT_TWO_FO_FC_TEMPLATE : template; + } else if (kind == DensityMapKind.FO_FC) { + foFcTemplate = template == null ? DEFAULT_FO_FC_TEMPLATE : template; + } else { + throw new IllegalArgumentException("Map coefficients exist only for 2Fo-Fc and Fo-Fc, not " + kind); + } + } + + /** Restores the default server and templates. */ + public static void resetToDefaults() { + serverBaseUrl = DEFAULT_SERVER_URL; + twoFoFcTemplate = DEFAULT_TWO_FO_FC_TEMPLATE; + foFcTemplate = DEFAULT_FO_FC_TEMPLATE; + } + + @Override + public DensityMapSource getSource() { + return DensityMapSource.WWPDB_MAP_COEFFICIENTS; + } + + @Override + public DensityFileFormat getFormat() { + return DensityFileFormat.MAP_COEFFICIENTS_CIF_GZ; + } + + @Override + public boolean supports(DensityMapKind kind) { + return kind == DensityMapKind.TWO_FO_FC || kind == DensityMapKind.FO_FC; + } + + /** + * Builds the URL for a set of coefficients without fetching them. + * + * @param pdbId the entry + * @param kind the kind of map + * @return the URL as a string + */ + public String buildUrl(PdbId pdbId, DensityMapKind kind) { + String template = kind == DensityMapKind.FO_FC ? foFcTemplate : twoFoFcTemplate; + return serverBaseUrl + UrlTemplates.expand(template, UrlTemplates.values(urlId(pdbId), null, -1)); + } + + @Override + public DensityMapResult fetch(DensityMapRequest request) throws IOException { + if (request.getPdbId() == null || !supports(request.getKind())) { + return null; + } + URL url = new URL(buildUrl(request.getPdbId(), request.getKind())); + File target = DensityCacheLayout.pdbMapFile(effectiveCacheRoot(request), request.getPdbId(), + request.getKind(), getSource(), getFormat(), null); + return obtain(request, url, target, request.getKind(), null, null, null); + } +} diff --git a/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestCcp4Header.java b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestCcp4Header.java new file mode 100644 index 0000000000..3c5d25950c --- /dev/null +++ b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestCcp4Header.java @@ -0,0 +1,112 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.zip.GZIPOutputStream; + +import org.biojava.nbio.core.util.FileDownloadUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * The CCP4 header check that keeps a server's error page out of the cache. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class TestCcp4Header { + + private File dir; + + @BeforeEach + public void setUp() throws IOException { + dir = Files.createTempDirectory("bj-ccp4").toFile(); + } + + @AfterEach + public void tearDown() throws IOException { + FileDownloadUtils.deleteDirectory(dir.toPath()); + } + + /** A minimal file carrying the stamp at the offset a real CCP4 header uses. */ + private static byte[] fakeMap() { + byte[] bytes = new byte[2048]; + byte[] stamp = Ccp4Header.MAP_STAMP.getBytes(StandardCharsets.US_ASCII); + System.arraycopy(stamp, 0, bytes, Ccp4Header.MAP_STAMP_OFFSET, stamp.length); + return bytes; + } + + private File write(String name, byte[] content) throws IOException { + File f = new File(dir, name); + Files.write(f.toPath(), content); + return f; + } + + @Test + public void recognisesAMapByItsStamp() throws IOException { + assertTrue(Ccp4Header.isCcp4(write("good.ccp4", fakeMap()))); + } + + @Test + public void recognisesAGzippedMap() throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + try (GZIPOutputStream gz = new GZIPOutputStream(buffer)) { + gz.write(fakeMap()); + } + assertTrue(Ccp4Header.isCcp4(write("good.map.gz", buffer.toByteArray())), "EMDB serves its maps gzipped"); + } + + /** + * The case this check exists for: a server answering with an error page and an + * HTTP 200, which nothing else would catch. + */ + @Test + public void rejectsAnHtmlErrorPage() throws IOException { + StringBuilder html = new StringBuilder("404 Not Found"); + while (html.length() < 1500) { + html.append("

The requested resource was not found on this server.

"); + } + html.append(""); + assertFalse(Ccp4Header.isCcp4(write("error.ccp4", html.toString().getBytes(StandardCharsets.UTF_8)))); + } + + @Test + public void rejectsRandomBytesAndShortFiles() throws IOException { + byte[] noise = new byte[2048]; + for (int i = 0; i < noise.length; i++) { + noise[i] = (byte) (i * 31); + } + assertFalse(Ccp4Header.isCcp4(write("noise.ccp4", noise))); + assertFalse(Ccp4Header.isCcp4(write("tiny.ccp4", new byte[10])), + "a file shorter than the header cannot be a map"); + } + + @Test + public void quietVariantSwallowsUnreadableFiles() { + assertFalse(Ccp4Header.isCcp4Quietly(new File(dir, "does-not-exist.ccp4"))); + } +} diff --git a/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestDensityCacheLayout.java b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestDensityCacheLayout.java new file mode 100644 index 0000000000..7e5723e67e --- /dev/null +++ b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestDensityCacheLayout.java @@ -0,0 +1,145 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.util.HashSet; +import java.util.Set; + +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.io.LocalPDBDirectory; +import org.junit.jupiter.api.Test; + +/** + * Cache layout: directory derivation, and that no two source and kind + * combinations can land on the same file. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class TestDensityCacheLayout { + + private static final File ROOT = new File("/tmp/bjcache"); + + /** + * The divided-archive directory is taken from the end of the identifier, not + * the start. That is what makes both spellings of an entry agree: counting from + * the start would file the extended form under "db" instead of "cb". + */ + @Test + public void middleHashIsTakenFromTheEndOfTheIdentifier() { + assertEquals("cb", LocalPDBDirectory.getMiddleHash("1cbs")); + assertEquals("cb", LocalPDBDirectory.getMiddleHash("1CBS")); + assertEquals("cb", LocalPDBDirectory.getMiddleHash("pdb_00001cbs")); + assertEquals("cb", LocalPDBDirectory.getMiddleHash("PDB_00001CBS")); + + // Guard against a regression to the substring(1, 3) form. + assertFalse("db".equals(LocalPDBDirectory.getMiddleHash("pdb_00001cbs"))); + } + + @Test + public void bothSpellingsOfAnEntryShareADirectory() { + File shortForm = DensityCacheLayout.pdbMapFile(ROOT, new PdbId("1cbs"), + DensityMapKind.TWO_FO_FC, DensityMapSource.PDBE_CCP4, DensityFileFormat.CCP4, null); + File extendedForm = DensityCacheLayout.pdbMapFile(ROOT, new PdbId("PDB_00001CBS"), + DensityMapKind.TWO_FO_FC, DensityMapSource.PDBE_CCP4, DensityFileFormat.CCP4, null); + assertEquals(shortForm, extendedForm); + } + + @Test + public void pdbMapFileNamesAreLowerCaseAndDivided() { + File f = DensityCacheLayout.pdbMapFile(ROOT, new PdbId("1CBS"), + DensityMapKind.TWO_FO_FC, DensityMapSource.PDBE_CCP4, DensityFileFormat.CCP4, null); + assertEquals("1cbs_2fofc_pdbe.ccp4", f.getName()); + assertEquals("cb", f.getParentFile().getName()); + assertEquals(DensityCacheLayout.DENSITY_DIR, f.getParentFile().getParentFile().getName()); + } + + @Test + public void qualifierDistinguishesDetailLevels() { + File d0 = DensityCacheLayout.pdbMapFile(ROOT, new PdbId("1cbs"), DensityMapKind.TWO_FO_FC, + DensityMapSource.RCSB_VOLUME_SERVER, DensityFileFormat.BCIF_VOLUME, "d0"); + File d3 = DensityCacheLayout.pdbMapFile(ROOT, new PdbId("1cbs"), DensityMapKind.TWO_FO_FC, + DensityMapSource.RCSB_VOLUME_SERVER, DensityFileFormat.BCIF_VOLUME, "d3"); + assertFalse(d0.equals(d3)); + assertTrue(d0.getName().endsWith("_d0.bcif")); + } + + /** + * Every combination has to map to a distinct file, or one source would serve a + * cache hit belonging to another. + */ + @Test + public void everySourceAndKindCombinationIsDistinct() { + Set seen = new HashSet<>(); + PdbId id = new PdbId("1cbs"); + for (DensityMapSource source : DensityMapSource.values()) { + for (DensityMapKind kind : DensityMapKind.values()) { + if (kind == DensityMapKind.AUTO || kind == DensityMapKind.EM) { + continue; // AUTO is never cached; EM is keyed by EMDB id instead + } + for (DensityFileFormat format : DensityFileFormat.values()) { + File f = DensityCacheLayout.pdbMapFile(ROOT, id, kind, source, format, null); + assertTrue(seen.add(f.getPath()), "duplicate cache path: " + f); + } + } + } + } + + @Test + public void emdbMapsAreKeyedByEmdbIdentifier() { + File f = DensityCacheLayout.emdbMapFile(ROOT, "emd-262", DensityMapSource.EMDB_MAP, + DensityFileFormat.CCP4_GZ, null); + assertEquals("emd_262.map.gz", f.getName()); + assertEquals("EMD-262", f.getParentFile().getName()); + assertEquals(DensityCacheLayout.EMDB_DIR, f.getParentFile().getParentFile().getName()); + } + + @Test + public void emdbIdentifiersAreNormalised() { + assertEquals("EMD-0262", DensityMapRequest.normalizeEmdbId("EMD-0262")); + assertEquals("EMD-0262", DensityMapRequest.normalizeEmdbId("emd-0262")); + assertEquals("EMD-0262", DensityMapRequest.normalizeEmdbId("EMD_0262")); + assertEquals("EMD-0262", DensityMapRequest.normalizeEmdbId("0262")); + assertEquals("0262", DensityMapRequest.emdbNumber("EMD-0262")); + } + + /** + * The marker has to sit inside the file name, before the extension. Jmol reads + * the difference block only when it finds that text in the name itself. + */ + @Test + public void differenceMarkerGoesInsideTheName() { + File plain = new File("/tmp/bjcache/density/cb/1cbs_both_rcsbvs_d3.bcif"); + File marked = DensityCacheLayout.differenceMarkerFile(plain); + assertEquals("1cbs_both_rcsbvs_d3&diff=1.bcif", marked.getName()); + assertEquals(plain.getAbsoluteFile().getParentFile(), marked.getParentFile()); + } + + @Test + public void emdbMappingFileIsDivided() { + File f = DensityCacheLayout.emdbMappingFile(ROOT, new PdbId("6hu9")); + assertEquals("6hu9.emdb.properties", f.getName()); + assertEquals("hu", f.getParentFile().getName()); + assertEquals(DensityCacheLayout.EMDB_MAPPING_DIR, f.getParentFile().getParentFile().getName()); + } +} diff --git a/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestDensityFallbackChain.java b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestDensityFallbackChain.java new file mode 100644 index 0000000000..ca33f34063 --- /dev/null +++ b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestDensityFallbackChain.java @@ -0,0 +1,291 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.biojava.nbio.core.util.HttpStatusException; +import org.biojava.nbio.structure.PdbId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * The fallback chain, exercised with stub providers so that no server is + * contacted. + *

+ * The behaviour that matters most here is the difference between "this source + * has nothing for the entry" and "this source could not be reached". The first + * must move on to the next source; the second must abort, because reporting a + * network outage as "no density exists" would be actively misleading. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class TestDensityFallbackChain { + + private DensityMapCache cache; + private List called; + + @BeforeEach + public void setUp() { + cache = new DensityMapCache(System.getProperty("java.io.tmpdir")); + called = new ArrayList<>(); + } + + /** A provider that behaves however the test needs it to. */ + private class StubProvider implements DensityMapProvider { + + private final DensityMapSource source; + private final DensityFileFormat format; + private final IOException failure; + private final boolean returnsNull; + + StubProvider(DensityMapSource source, DensityFileFormat format, IOException failure, boolean returnsNull) { + this.source = source; + this.format = format; + this.failure = failure; + this.returnsNull = returnsNull; + } + + @Override + public DensityMapSource getSource() { + return source; + } + + @Override + public DensityFileFormat getFormat() { + return format; + } + + @Override + public boolean supports(DensityMapKind kind) { + return kind != DensityMapKind.AUTO; + } + + @Override + public DensityMapResult fetch(DensityMapRequest request) throws IOException { + called.add(source); + if (failure != null) { + throw failure; + } + if (returnsNull) { + return null; + } + return new DensityMapResult(new File("stub.map"), source, format, request.getKind(), + request.getPdbId(), request.getEmdbId(), "stub://" + source, false, null, null); + } + } + + private StubProvider missing(DensityMapSource source) { + return new StubProvider(source, DensityFileFormat.CCP4, + new HttpStatusException(404, "stub://" + source, "Not Found"), false); + } + + private StubProvider succeeds(DensityMapSource source) { + return new StubProvider(source, DensityFileFormat.CCP4, null, false); + } + + private void useOnly(DensityMapSource... sources) { + cache.setSourceChain(DensityMapKind.TWO_FO_FC, Arrays.asList(sources)); + for (DensityMapSource s : DensityMapSource.values()) { + cache.setSourceEnabled(s, Arrays.asList(sources).contains(s)); + } + } + + @Test + public void aMissingSourceFallsThroughToTheNext() throws IOException { + cache.registerProvider(missing(DensityMapSource.RCSB_VOLUME_SERVER)); + cache.registerProvider(succeeds(DensityMapSource.PDBE_CCP4)); + useOnly(DensityMapSource.RCSB_VOLUME_SERVER, DensityMapSource.PDBE_CCP4); + + DensityMapResult result = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + + assertEquals(DensityMapSource.PDBE_CCP4, result.getSource()); + assertEquals(Arrays.asList(DensityMapSource.RCSB_VOLUME_SERVER, DensityMapSource.PDBE_CCP4), called); + } + + @Test + public void returningNullAlsoFallsThrough() throws IOException { + cache.registerProvider(new StubProvider(DensityMapSource.RCSB_VOLUME_SERVER, + DensityFileFormat.CCP4, null, true)); + cache.registerProvider(succeeds(DensityMapSource.PDBE_CCP4)); + useOnly(DensityMapSource.RCSB_VOLUME_SERVER, DensityMapSource.PDBE_CCP4); + + assertEquals(DensityMapSource.PDBE_CCP4, + cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC).getSource()); + } + + @Test + public void aTooLargeMapFallsThroughToASmallerSource() throws IOException { + cache.registerProvider(new StubProvider(DensityMapSource.EMDB_MAP, DensityFileFormat.CCP4_GZ, + new DensityMapTooLargeException("stub://big", 111426503L, 1024L), false)); + cache.registerProvider(succeeds(DensityMapSource.RCSB_VOLUME_SERVER)); + useOnly(DensityMapSource.EMDB_MAP, DensityMapSource.RCSB_VOLUME_SERVER); + + assertEquals(DensityMapSource.RCSB_VOLUME_SERVER, + cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC).getSource()); + } + + /** + * A dropped connection is not evidence that the entry has no density, so the + * chain must stop rather than quietly try the rest and report "none available". + */ + @Test + public void aTransportFailureAbortsTheChain() { + cache.registerProvider(new StubProvider(DensityMapSource.RCSB_VOLUME_SERVER, DensityFileFormat.CCP4, + new SocketTimeoutException("connection timed out"), false)); + cache.registerProvider(succeeds(DensityMapSource.PDBE_CCP4)); + useOnly(DensityMapSource.RCSB_VOLUME_SERVER, DensityMapSource.PDBE_CCP4); + + try { + cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + fail("a transport failure should not be reported as a missing map"); + } catch (NoDensityMapException e) { + fail("a transport failure must not be reported as NoDensityMapException"); + } catch (IOException expected) { + assertEquals(Arrays.asList(DensityMapSource.RCSB_VOLUME_SERVER), called); + } + } + + /** A 5xx is a server problem, not an absent entry, so it must abort too. */ + @Test + public void aServerErrorAbortsTheChain() { + cache.registerProvider(new StubProvider(DensityMapSource.RCSB_VOLUME_SERVER, DensityFileFormat.CCP4, + new HttpStatusException(503, "stub://x", "Service Unavailable"), false)); + cache.registerProvider(succeeds(DensityMapSource.PDBE_CCP4)); + useOnly(DensityMapSource.RCSB_VOLUME_SERVER, DensityMapSource.PDBE_CCP4); + + try { + cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + fail("HTTP 503 should abort the chain"); + } catch (IOException expected) { + assertTrue(expected instanceof HttpStatusException); + assertEquals(503, ((HttpStatusException) expected).getStatusCode()); + } + } + + @Test + public void exhaustingEverySourceReportsWhatWasTried() { + cache.registerProvider(missing(DensityMapSource.RCSB_VOLUME_SERVER)); + cache.registerProvider(missing(DensityMapSource.PDBE_CCP4)); + useOnly(DensityMapSource.RCSB_VOLUME_SERVER, DensityMapSource.PDBE_CCP4); + + try { + cache.getDensityMap(new PdbId("4hhb"), DensityMapKind.TWO_FO_FC); + fail("expected NoDensityMapException"); + } catch (NoDensityMapException e) { + assertEquals(2, e.getAttempts().size()); + assertTrue(e.getAttempts().get(DensityMapSource.RCSB_VOLUME_SERVER).contains("404")); + assertTrue(e.getMessage().contains("4HHB") || e.getMessage().contains("4hhb")); + } catch (IOException e) { + fail("expected NoDensityMapException but got " + e); + } + } + + @Test + public void aDisabledSourceIsNotCalled() { + cache.registerProvider(succeeds(DensityMapSource.PDBE_CCP4)); + useOnly(DensityMapSource.PDBE_CCP4); + cache.setSourceEnabled(DensityMapSource.PDBE_CCP4, false); + + try { + cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + fail("expected NoDensityMapException"); + } catch (NoDensityMapException e) { + assertTrue(called.isEmpty()); + assertEquals("disabled", e.getAttempts().get(DensityMapSource.PDBE_CCP4)); + } catch (IOException e) { + fail("expected NoDensityMapException but got " + e); + } + } + + /** Map coefficients are archival only, so they are off unless asked for. */ + @Test + public void mapCoefficientsAreDisabledByDefault() { + assertFalse(new DensityMapCache(System.getProperty("java.io.tmpdir")) + .isSourceEnabled(DensityMapSource.WWPDB_MAP_COEFFICIENTS)); + } + + /** + * A viewer asks for renderable formats only; that must exclude the coefficients + * even when the source is enabled. + */ + @Test + public void unrenderableFormatsAreSkippedWhenTheCallerCannotUseThem() { + cache.registerProvider(new StubProvider(DensityMapSource.WWPDB_MAP_COEFFICIENTS, + DensityFileFormat.MAP_COEFFICIENTS_CIF_GZ, null, false)); + useOnly(DensityMapSource.WWPDB_MAP_COEFFICIENTS); + + try { + cache.getDensityMap(DensityMapRequest.builder(new PdbId("1cbs")) + .kind(DensityMapKind.TWO_FO_FC) + .allowNonRenderableFormats(false) + .build()); + fail("expected NoDensityMapException"); + } catch (NoDensityMapException e) { + assertTrue(called.isEmpty()); + assertTrue(e.getAttempts().get(DensityMapSource.WWPDB_MAP_COEFFICIENTS).contains("Fourier")); + } catch (IOException e) { + fail("expected NoDensityMapException but got " + e); + } + } + + @Test + public void coefficientsAreUsedWhenTheCallerAllowsThem() throws IOException { + cache.registerProvider(new StubProvider(DensityMapSource.WWPDB_MAP_COEFFICIENTS, + DensityFileFormat.MAP_COEFFICIENTS_CIF_GZ, null, false)); + useOnly(DensityMapSource.WWPDB_MAP_COEFFICIENTS); + + DensityMapResult result = cache.getDensityMap(DensityMapRequest.builder(new PdbId("1cbs")) + .kind(DensityMapKind.TWO_FO_FC) + .allowNonRenderableFormats(true) + .build()); + assertFalse(result.isRenderable()); + } + + /** AUTO tries the X-ray map first, then falls through to the EM one. */ + @Test + public void autoFallsFromXrayToEm() { + assertEquals(Arrays.asList(DensityMapKind.TWO_FO_FC, DensityMapKind.EM), + DensityMapKind.AUTO.resolve()); + assertEquals(Arrays.asList(DensityMapKind.FO_FC), DensityMapKind.FO_FC.resolve()); + } + + /** X-ray and EM entries are served by deliberately different orders. */ + @Test + public void defaultChainsPreferTheSmallestSource() { + DensityMapCache fresh = new DensityMapCache(System.getProperty("java.io.tmpdir")); + assertEquals(DensityMapSource.RCSB_VOLUME_SERVER, + fresh.getSourceChain(DensityMapKind.TWO_FO_FC).get(0)); + assertEquals(DensityMapSource.RCSB_VOLUME_SERVER, + fresh.getSourceChain(DensityMapKind.EM).get(0)); + // the full-resolution archive is the last resort for EM + List em = fresh.getSourceChain(DensityMapKind.EM); + assertEquals(DensityMapSource.EMDB_MAP, em.get(em.size() - 1)); + } +} diff --git a/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestDensityMapUrlTemplates.java b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestDensityMapUrlTemplates.java new file mode 100644 index 0000000000..716b02b632 --- /dev/null +++ b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestDensityMapUrlTemplates.java @@ -0,0 +1,235 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; + +import org.biojava.nbio.structure.PdbId; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * URL construction for each provider. These run offline: only the strings are + * built, nothing is fetched. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class TestDensityMapUrlTemplates { + + private static final File ROOT = new File("/tmp/bjcache"); + + @AfterEach + public void restoreDefaults() { + PdbeCcp4MapProvider.resetToDefaults(); + WwpdbMapCoefficientsProvider.resetToDefaults(); + EmdbMapProvider.resetToDefaults(); + } + + /** + * PDBe answers only to the lower-case four-character spelling; an upper-case or + * extended one returns HTTP 404. + */ + @Test + public void pdbeUrlsAreLowerCase() { + PdbeCcp4MapProvider p = new PdbeCcp4MapProvider(ROOT); + assertEquals("https://www.ebi.ac.uk/pdbe/coordinates/files/1cbs.ccp4", + p.buildUrl(new PdbId("1CBS"), DensityMapKind.TWO_FO_FC)); + assertEquals("https://www.ebi.ac.uk/pdbe/coordinates/files/1cbs_diff.ccp4", + p.buildUrl(new PdbId("1cbs"), DensityMapKind.FO_FC)); + } + + @Test + public void pdbeServerIsConfigurable() { + PdbeCcp4MapProvider.setServerBaseUrl("http://localhost:1/maps"); + PdbeCcp4MapProvider p = new PdbeCcp4MapProvider(ROOT); + assertEquals("http://localhost:1/maps/1cbs.ccp4", + p.buildUrl(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC)); + } + + /** + * The coefficients are addressed by file name, not by a constructed directory + * path, so that the July 2027 move to the per-entry archive does not invalidate + * the URLs. + */ + @Test + public void wwpdbUrlsResolveByName() { + WwpdbMapCoefficientsProvider p = new WwpdbMapCoefficientsProvider(ROOT); + assertEquals("https://files.wwpdb.org/validation/download/" + + "1cbs_validation_2fo-fc_map_coef.cif.gz", + p.buildUrl(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC)); + assertEquals("https://files.wwpdb.org/validation/download/" + + "1cbs_validation_fo-fc_map_coef.cif.gz", + p.buildUrl(new PdbId("1cbs"), DensityMapKind.FO_FC)); + } + + /** + * The identifier spelling is not hard-coded either way. An entry with a short + * form is asked for by it; one without — every entry deposited once the + * four-character space is exhausted — is asked for by its extended form. + * Both spellings resolve at the endpoint, so nothing here has to change in 2027. + */ + @Test + public void wwpdbUrlsFollowWhicheverSpellingTheEntryHas() { + WwpdbMapCoefficientsProvider p = new WwpdbMapCoefficientsProvider(ROOT); + assertTrue(p.buildUrl(new PdbId("pdb_00001cbs"), DensityMapKind.TWO_FO_FC) + .endsWith("/1cbs_validation_2fo-fc_map_coef.cif.gz"), + "a shortable entry is asked for by its short name"); + assertTrue(p.buildUrl(new PdbId("pdb_00012abc"), DensityMapKind.TWO_FO_FC) + .endsWith("/pdb_00012abc_validation_2fo-fc_map_coef.cif.gz"), + "an entry with no short form is asked for by its extended name"); + } + + /** + * The beta host is a host swap and nothing more: the same endpoint, the same + * file name. That is what makes it a test target rather than a default - it + * holds the post-2027 content, so a request against it exercises the archive as + * it will be, while the default host is the one that survives the cutover. + */ + @Test + public void betaHostChangesNothingButTheHost() { + WwpdbMapCoefficientsProvider p = new WwpdbMapCoefficientsProvider(ROOT); + String viaDefault = p.buildUrl(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + WwpdbMapCoefficientsProvider.setServerBaseUrl(WwpdbMapCoefficientsProvider.BETA_SERVER_URL); + String viaBeta = p.buildUrl(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + assertEquals(viaDefault.substring(viaDefault.lastIndexOf('/')), + viaBeta.substring(viaBeta.lastIndexOf('/'))); + assertTrue(viaBeta.startsWith("https://files-beta.wwpdb.org/validation/download/")); + } + + /** EBI publishes directories rather than an endpoint, so it needs the divided templates. */ + @Test + public void dividedTemplatesStillReachMirrors() { + WwpdbMapCoefficientsProvider.setServerBaseUrl(WwpdbMapCoefficientsProvider.EBI_MIRROR_URL); + WwpdbMapCoefficientsProvider.setPathUrlTemplate(DensityMapKind.TWO_FO_FC, + WwpdbMapCoefficientsProvider.DIVIDED_TWO_FO_FC_TEMPLATE); + WwpdbMapCoefficientsProvider p = new WwpdbMapCoefficientsProvider(ROOT); + assertEquals("https://ftp.ebi.ac.uk/pub/databases/pdb/validation_reports/" + + "cb/1cbs/1cbs_validation_2fo-fc_map_coef.cif.gz", + p.buildUrl(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC)); + } + + /** + * The archive layout that arrives in July 2027 is expressible as a template, + * so a mirror of it needs configuration rather than a new release. + */ + @Test + public void perEntryTemplateUsesExtendedIdentifiers() { + WwpdbMapCoefficientsProvider.setServerBaseUrl("https://files-beta.wwpdb.org/pub/wwpdb/pdb/data/"); + WwpdbMapCoefficientsProvider.setPathUrlTemplate(DensityMapKind.TWO_FO_FC, + WwpdbMapCoefficientsProvider.ENTRIES_TWO_FO_FC_TEMPLATE); + WwpdbMapCoefficientsProvider p = new WwpdbMapCoefficientsProvider(ROOT); + assertEquals("https://files-beta.wwpdb.org/pub/wwpdb/pdb/data/" + + "entries/cb/pdb_00001cbs/validation_reports/" + + "pdb_00001cbs_validation_2fo-fc_map_coef.cif.gz", + p.buildUrl(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC)); + } + + /** The extended spelling is available to any template, in the case the archive uses. */ + @Test + public void extendedIdentifierPlaceholder() { + assertEquals("pdb_00001cbs", UrlTemplates.values("1cbs", null, -1).get("extid")); + assertEquals("pdb_00001cbs", UrlTemplates.values("pdb_00001cbs", null, -1).get("extid")); + assertEquals("{extid}", + UrlTemplates.expand("{extid}", UrlTemplates.values("not an id", null, -1)), + "an unusable identifier leaves the placeholder alone rather than guessing"); + } + + /** + * The two-character directory is counted from the right hand end, so it is the + * same for both spellings. This is what lets the cache layout stay as it is + * across the 2027 transition, and it is the rule the wwPDB documents for the + * new archive. + */ + @Test + public void middleHashIsTheSameForBothSpellings() { + assertEquals("cb", UrlTemplates.values("1cbs", null, -1).get("mid")); + assertEquals("cb", UrlTemplates.values("pdb_00001cbs", null, -1).get("mid")); + } + + @Test + public void volumeServerUrlsCarryDetailAndEncoding() { + VolumeServerProvider rcsb = new VolumeServerProvider(ROOT, VolumeServerProvider.Host.RCSB); + rcsb.setDetail(0); + String url = rcsb.buildUrl(DensityMapRequest.builder(new PdbId("1cbs")) + .kind(DensityMapKind.TWO_FO_FC).build()); + assertEquals("https://maps.rcsb.org/x-ray/1cbs/cell?detail=0&encoding=bcif", url); + + VolumeServerProvider pdbe = new VolumeServerProvider(ROOT, VolumeServerProvider.Host.PDBE); + pdbe.setDetail(6); + assertEquals("https://www.ebi.ac.uk/pdbe/volume-server/x-ray/1cbs/cell?detail=6&encoding=bcif", + pdbe.buildUrl(DensityMapRequest.builder(new PdbId("1cbs")) + .kind(DensityMapKind.TWO_FO_FC).build())); + } + + @Test + public void volumeServerEmUrlUsesTheEmdbNumber() { + VolumeServerProvider rcsb = new VolumeServerProvider(ROOT, VolumeServerProvider.Host.RCSB); + rcsb.setDetail(3); + String url = rcsb.buildUrl(DensityMapRequest.builder("EMD-0262").build()); + assertEquals("https://maps.rcsb.org/em/emd-0262/cell?detail=3&encoding=bcif", url); + } + + @Test + public void encodingSelectsTheFormat() { + VolumeServerProvider p = new VolumeServerProvider(ROOT, VolumeServerProvider.Host.RCSB); + assertEquals(DensityFileFormat.BCIF_VOLUME, p.getFormat()); + p.setEncoding("cif"); + assertEquals(DensityFileFormat.CIF_VOLUME, p.getFormat()); + assertTrue(p.buildUrl(DensityMapRequest.builder(new PdbId("1cbs")) + .kind(DensityMapKind.TWO_FO_FC).build()).endsWith("encoding=cif")); + } + + @Test + public void emdbMapUrl() { + EmdbMapProvider p = new EmdbMapProvider(ROOT, null); + assertEquals("https://ftp.ebi.ac.uk/pub/databases/emdb/structures/EMD-0262/map/emd_0262.map.gz", + p.buildUrl("EMD-0262")); + assertEquals("https://ftp.ebi.ac.uk/pub/databases/emdb/structures/EMD-0262/map/emd_0262.map.gz", + p.buildUrl("emd_0262")); + } + + /** An unknown placeholder must survive verbatim, so a bad template is obvious. */ + @Test + public void unknownPlaceholdersAreLeftAlone() { + assertEquals("a/{nosuch}/b", + UrlTemplates.expand("a/{nosuch}/b", UrlTemplates.values("1cbs", null, -1))); + } + + @Test + public void providersDeclareWhatTheyCanServe() { + assertTrue(new PdbeCcp4MapProvider(ROOT).supports(DensityMapKind.TWO_FO_FC)); + assertTrue(!new PdbeCcp4MapProvider(ROOT).supports(DensityMapKind.EM)); + assertTrue(new EmdbMapProvider(ROOT, null).supports(DensityMapKind.EM)); + assertTrue(!new EmdbMapProvider(ROOT, null).supports(DensityMapKind.TWO_FO_FC)); + assertTrue(new VolumeServerProvider(ROOT, VolumeServerProvider.Host.RCSB).supports(DensityMapKind.EM)); + } + + /** Only the coefficients are unrenderable; every sampled grid format is fine. */ + @Test + public void onlyCoefficientsAreUnrenderable() { + assertTrue(!DensityFileFormat.MAP_COEFFICIENTS_CIF_GZ.isJmolLoadable()); + assertTrue(DensityFileFormat.CCP4.isJmolLoadable()); + assertTrue(DensityFileFormat.CCP4_GZ.isJmolLoadable()); + assertTrue(DensityFileFormat.BCIF_VOLUME.isJmolLoadable()); + assertTrue(DensityFileFormat.CIF_VOLUME.isJmolLoadable()); + } +} diff --git a/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestEmdbAndLocalOnly.java b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestEmdbAndLocalOnly.java new file mode 100644 index 0000000000..97cef78926 --- /dev/null +++ b/biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestEmdbAndLocalOnly.java @@ -0,0 +1,221 @@ +/** + * BioJava development code + * + * This code may be freely distributed and modified under the terms of the GNU + * Lesser General Public Licence. This should be distributed with the code. If + * you do not have a copy, see: + * + * http://www.gnu.org/copyleft/lesser.html + * + * Copyright for this code is held jointly by the individual authors. These + * should be listed in @author doc comments. + * + * For more information on the BioJava project and its aims, or to join the + * biojava-l mailing list, visit the home page at: + * + * http://www.biojava.org/ + */ +package org.biojava.nbio.structure.io.density; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.List; + +import org.biojava.nbio.core.util.FileDownloadUtils; +import org.biojava.nbio.structure.PdbId; +import org.biojava.nbio.structure.io.LocalPDBDirectory.FetchBehavior; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * EMDB metadata parsing against captured responses, and the guarantee that + * LOCAL_ONLY never opens a connection. + * + * @author Amr ALHOSSARY + * @since 7.3.0 + */ +public class TestEmdbAndLocalOnly { + + /** + * Nothing listens here, so any attempt to reach a server fails immediately + * rather than hanging or, worse, quietly succeeding. + */ + private static final String DEAD_SERVER = "http://localhost:1/"; + + private File cacheRoot; + + @BeforeEach + public void setUp() throws IOException { + cacheRoot = Files.createTempDirectory("bj-density").toFile(); + } + + @AfterEach + public void tearDown() throws IOException { + FileDownloadUtils.deleteDirectory(cacheRoot.toPath()); + PdbeCcp4MapProvider.resetToDefaults(); + WwpdbMapCoefficientsProvider.resetToDefaults(); + EmdbMapProvider.resetToDefaults(); + EmdbEntryResolver.resetToDefaults(); + } + + private void copyResource(String resource, File target) throws IOException { + target.getParentFile().mkdirs(); + try (InputStream in = getClass().getResourceAsStream(resource)) { + assertNotNull(in, "missing test resource " + resource); + Files.copy(in, target.toPath()); + } + } + + /** + * The map metadata carries the two numbers a viewer needs: the level the + * depositors recommend and the RMS deviation that converts it to sigma. + */ + @Test + public void parsesEmdbMapMetadata() throws IOException { + copyResource("emdb-map-EMD-0262.json", + DensityCacheLayout.emdbMapInfoFile(cacheRoot, "EMD-0262")); + + EmdbEntryResolver resolver = new EmdbEntryResolver(cacheRoot); + resolver.setFetchBehavior(FetchBehavior.LOCAL_ONLY); + EmdbEntryInfo info = resolver.getEntryInfo("EMD-0262"); + + assertNotNull(info); + assertEquals("EMD-0262", info.getEmdbId()); + assertEquals(0.0263, info.getRecommendedContourLevel(), 1e-9); + assertNotNull(info.getSigma(), "sigma is needed to express the contour in sigma units"); + // 119165 kB, which is well past the default download ceiling + assertEquals(119165L * 1024L, info.getMapSizeBytes().longValue()); + } + + /** + * The size of a real EM map, and why the source order rather than the size + * guard is what keeps it from being fetched. + *

+ * EMD-0262 is about 116 MB, which sits comfortably under the 256 MiB default + * ceiling: left to the ceiling alone it would be downloaded in full. What + * avoids that is putting the density servers ahead of the archive, which serve + * a few megabytes for the same entry. The ceiling is the backstop for the maps + * that run to gigabytes. + */ + @Test + public void theFullEmMapIsLargeButWithinTheDefaultCeiling() throws IOException { + copyResource("emdb-map-EMD-0262.json", + DensityCacheLayout.emdbMapInfoFile(cacheRoot, "EMD-0262")); + EmdbEntryResolver resolver = new EmdbEntryResolver(cacheRoot); + resolver.setFetchBehavior(FetchBehavior.LOCAL_ONLY); + + long bytes = resolver.getEntryInfo("EMD-0262").getMapSizeBytes(); + assertTrue(bytes > 100L * 1024 * 1024, "expected a map of order 100 MB, got " + bytes); + assertTrue(bytes < DensityMapCache.DEFAULT_MAX_DOWNLOAD_BYTES, + "the ceiling alone would not stop this download"); + + DensityMapCache cache = new DensityMapCache(cacheRoot.getAbsolutePath()); + List em = cache.getSourceChain(DensityMapKind.EM); + assertTrue(em.indexOf(DensityMapSource.RCSB_VOLUME_SERVER) < em.indexOf(DensityMapSource.EMDB_MAP), + "a density server must be tried before the full archive"); + } + + @Test + public void contourConvertsToSigma() throws IOException { + copyResource("emdb-map-EMD-0262.json", + DensityCacheLayout.emdbMapInfoFile(cacheRoot, "EMD-0262")); + EmdbEntryResolver resolver = new EmdbEntryResolver(cacheRoot); + resolver.setFetchBehavior(FetchBehavior.LOCAL_ONLY); + EmdbEntryInfo info = resolver.getEntryInfo("EMD-0262"); + + DensityMapResult result = new DensityMapResult(new File("x.map"), DensityMapSource.EMDB_MAP, + DensityFileFormat.CCP4_GZ, DensityMapKind.EM, new PdbId("6hu9"), "EMD-0262", "u", false, + info.getRecommendedContourLevel(), info.getSigma()); + + assertEquals(info.getRecommendedContourLevel() / info.getSigma(), result.getContourInSigma(), 1e-9); + } + + /** A cached mapping is used whatever its age when downloads are off. */ + @Test + public void localOnlyUsesACachedMappingWithoutAskingAnyServer() throws IOException { + File mappingFile = DensityCacheLayout.emdbMappingFile(cacheRoot, new PdbId("6hu9")); + mappingFile.getParentFile().mkdirs(); + Files.write(mappingFile.toPath(), + ("pdbId=6hu9\nemdbIds=EMD-0262\ncontourLevel=0.0263\nretrieved=1999-01-01T00:00:00Z\n") + .getBytes(StandardCharsets.UTF_8)); + + EmdbEntryResolver resolver = new EmdbEntryResolver(cacheRoot); + resolver.setFetchBehavior(FetchBehavior.LOCAL_ONLY); + // point every template at a dead port, so a lookup would fail loudly + EmdbEntryResolver.setSearchUrlTemplate(DEAD_SERVER + "{pdbid_lc}"); + EmdbEntryResolver.setRcsbEntryUrlTemplate(DEAD_SERVER + "{pdbid_lc}"); + + List ids = resolver.getEmdbIds(new PdbId("6hu9")); + assertEquals(1, ids.size()); + assertEquals("EMD-0262", ids.get(0)); + } + + @Test + public void localOnlyReturnsNothingWhenNothingIsCached() { + EmdbEntryResolver resolver = new EmdbEntryResolver(cacheRoot); + resolver.setFetchBehavior(FetchBehavior.LOCAL_ONLY); + EmdbEntryResolver.setSearchUrlTemplate(DEAD_SERVER + "{pdbid_lc}"); + EmdbEntryResolver.setRcsbEntryUrlTemplate(DEAD_SERVER + "{pdbid_lc}"); + + assertTrue(resolver.getEmdbIds(new PdbId("6hu9")).isEmpty()); + } + + /** + * A cached map is served without any network access at all, and its metadata is + * rebuilt from the sidecar rather than from a server. + */ + @Test + public void localOnlyServesACachedMapOffline() throws IOException { + DensityMapCache cache = new DensityMapCache(cacheRoot.getAbsolutePath()); + cache.setFetchBehavior(FetchBehavior.LOCAL_ONLY); + PdbeCcp4MapProvider.setServerBaseUrl(DEAD_SERVER); + cache.setSourceEnabled(DensityMapSource.RCSB_VOLUME_SERVER, false); + cache.setSourceEnabled(DensityMapSource.PDBE_VOLUME_SERVER, false); + + File cached = DensityCacheLayout.pdbMapFile(cacheRoot, new PdbId("1cbs"), + DensityMapKind.TWO_FO_FC, DensityMapSource.PDBE_CCP4, DensityFileFormat.CCP4, null); + cached.getParentFile().mkdirs(); + Files.write(cached.toPath(), fakeCcp4()); + + DensityMapResult result = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + + assertEquals(DensityMapSource.PDBE_CCP4, result.getSource()); + assertTrue(result.isFromCache()); + assertEquals(cached, result.getFile()); + assertTrue(DensityMapResult.metaFileFor(cached).isFile(), + "a sidecar should have been written for the recovered file"); + } + + @Test + public void localOnlyRefusesWhenNothingIsCached() { + DensityMapCache cache = new DensityMapCache(cacheRoot.getAbsolutePath()); + cache.setFetchBehavior(FetchBehavior.LOCAL_ONLY); + PdbeCcp4MapProvider.setServerBaseUrl(DEAD_SERVER); + + try { + cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC); + fail("expected NoDensityMapException"); + } catch (NoDensityMapException e) { + assertFalse(e.getAttempts().isEmpty()); + } catch (IOException e) { + fail("LOCAL_ONLY must not attempt any connection, but got " + e); + } + } + + private static byte[] fakeCcp4() { + byte[] bytes = new byte[4096]; + byte[] stamp = Ccp4Header.MAP_STAMP.getBytes(StandardCharsets.US_ASCII); + System.arraycopy(stamp, 0, bytes, Ccp4Header.MAP_STAMP_OFFSET, stamp.length); + return bytes; + } +} diff --git a/biojava-structure/src/test/resources/org/biojava/nbio/structure/io/density/emdb-map-EMD-0262.json b/biojava-structure/src/test/resources/org/biojava/nbio/structure/io/density/emdb-map-EMD-0262.json new file mode 100644 index 0000000000..3067838be7 --- /dev/null +++ b/biojava-structure/src/test/resources/org/biojava/nbio/structure/io/density/emdb-map-EMD-0262.json @@ -0,0 +1 @@ +{"emdb_id": "EMD-0262", "map": {"format": "CCP4", "size_kbytes": 119165, "file": "emd_0262.map.gz", "symmetry": {"space_group": "1"}, "data_type": "IMAGE STORED AS FLOATING POINT NUMBER (4 BYTES)", "dimensions": {"col": 310, "row": 310, "sec": 310}, "origin": {"col": 0, "row": 0, "sec": 0}, "spacing": {"x": 310, "y": 310, "z": 310}, "cell": {"a": {"units": "\u212b", "valueOf_": "429.691"}, "b": {"units": "\u212b", "valueOf_": "429.691"}, "c": {"units": "\u212b", "valueOf_": "429.691"}, "alpha": {"units": "deg", "valueOf_": "90.0"}, "beta": {"units": "deg", "valueOf_": "90.0"}, "gamma": {"units": "deg", "valueOf_": "90.0"}}, "axis_order": {"fast": "X", "medium": "Y", "slow": "Z"}, "statistics": {"minimum": -0.14692926, "maximum": 0.2550798, "average": 0.0005225369, "std": 0.009231779}, "pixel_spacing": {"x": {"units": "\u212b", "valueOf_": "1.3861"}, "y": {"units": "\u212b", "valueOf_": "1.3861"}, "z": {"units": "\u212b", "valueOf_": "1.3861"}}, "contour_list": {"contour": [{"primary": true, "level": 0.0263, "source": "AUTHOR", "instance_type": "contour"}]}, "label": "::::EMDATABANK.org::::EMD-0262::::", "annotation_details": "The sharpened map of the III2IV2 supercomplex"}} \ No newline at end of file diff --git a/biojava-structure/src/test/resources/org/biojava/nbio/structure/io/density/emdb-search-6hu9.csv b/biojava-structure/src/test/resources/org/biojava/nbio/structure/io/density/emdb-search-6hu9.csv new file mode 100644 index 0000000000..8046fd8b78 --- /dev/null +++ b/biojava-structure/src/test/resources/org/biojava/nbio/structure/io/density/emdb-search-6hu9.csv @@ -0,0 +1,2 @@ +emdb_id,map_contour_level_value +EMD-0262,0.0263 diff --git a/biojava-structure/src/test/resources/org/biojava/nbio/structure/io/density/rcsb-entry-6hu9.json b/biojava-structure/src/test/resources/org/biojava/nbio/structure/io/density/rcsb-entry-6hu9.json new file mode 100644 index 0000000000..b200c1817f --- /dev/null +++ b/biojava-structure/src/test/resources/org/biojava/nbio/structure/io/density/rcsb-entry-6hu9.json @@ -0,0 +1 @@ +{"audit_author":[{"identifier_ORCID":"0000-0001-7986-6736","name":"Hartley, A.M.","pdbx_ordinal":1},{"identifier_ORCID":"0000-0002-5096-257X","name":"Pinotsis, N.","pdbx_ordinal":2},{"identifier_ORCID":"0000-0003-3460-3806","name":"Marechal, A.","pdbx_ordinal":3}],"citation":[{"country":"US","id":"primary","journal_abbrev":"Nat. Struct. Mol. Biol.","journal_id_ISSN":"1545-9985","journal_volume":"26","page_first":"78","page_last":"83","pdbx_database_id_DOI":"10.1038/s41594-018-0172-z","pdbx_database_id_PubMed":30598554,"rcsb_authors":["Hartley, A.M.","Lukoyanova, N.","Zhang, Y.","Cabrera-Orefice, A.","Arnold, S.","Meunier, B.","Pinotsis, N.","Marechal, A."],"rcsb_is_primary":"Y","rcsb_journal_abbrev":"Nat Struct Mol Biol","title":"Structure of yeast cytochrome c oxidase in a supercomplex with cytochrome bc1.","year":2019}],"database_2":[{"database_code":"6HU9","database_id":"PDB","pdbx_DOI":"10.2210/pdb6hu9/pdb","pdbx_database_accession":"pdb_00006hu9"},{"database_code":"D_1200012277","database_id":"WWPDB"}],"em_3d_fitting":[{"id":"1","ref_protocol":"RIGID BODY FIT","ref_space":"REAL"}],"em_3d_fitting_list":[{"3d_fitting_id":"1","id":"1","pdb_entry_id":"1KYO"},{"3d_fitting_id":"1","id":"2","pdb_entry_id":"1V54"}],"em_3d_reconstruction":[{"id":"1","image_processing_id":"1","num_particles":44915,"resolution":3.35,"resolution_method":"FSC 0.5 CUT-OFF","symmetry_type":"POINT"}],"em_ctf_correction":[{"em_image_processing_id":"1","id":"1","type":"NONE"}],"em_entity_assembly":[{"entity_id_list":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22"],"id":"1","name":"III2-IV2 mitochondrial respiratory supercomplex","parent_id":0,"source":"NATURAL","type":"COMPLEX"}],"em_experiment":{"aggregation_state":"PARTICLE","entity_assembly_id":"1","id":"1","reconstruction_method":"SINGLE PARTICLE"},"em_image_recording":[{"average_exposure_time":8.0,"avg_electron_dose_per_image":1.645,"detector_mode":"COUNTING","film_or_detector_model":"GATAN K2 QUANTUM (4k x 4k)","id":"1","imaging_id":"1"}],"em_imaging":[{"accelerating_voltage":300,"c2_aperture_diameter":70.0,"cryogen":"NITROGEN","electron_source":"FIELD EMISSION GUN","id":"1","illumination_mode":"FLOOD BEAM","microscope_model":"FEI TITAN KRIOS","mode":"BRIGHT FIELD","nominal_magnification":130000,"specimen_id":"1"}],"em_single_particle_entity":[{"id":1,"image_processing_id":"1","point_symmetry":"C1"}],"em_software":[{"category":"PARTICLE SELECTION","id":"1","image_processing_id":"1","name":"Gautomatch","version":"0.53"},{"category":"IMAGE ACQUISITION","id":"2","imaging_id":"1","name":"EPU"},{"category":"MASKING","id":"3"},{"category":"CTF CORRECTION","id":"4","image_processing_id":"1","name":"CTFFIND"},{"category":"LAYERLINE INDEXING","id":"5"},{"category":"DIFFRACTION INDEXING","id":"6"},{"category":"MODEL FITTING","fitting_id":"1","id":"7","name":"UCSF Chimera"},{"category":"OTHER","id":"8"},{"category":"MODEL REFINEMENT","fitting_id":"1","id":"9","name":"PHENIX","version":"1.13_2998"},{"category":"INITIAL EULER ASSIGNMENT","id":"10","image_processing_id":"1","name":"RELION","version":"3.0"},{"category":"FINAL EULER ASSIGNMENT","id":"11","image_processing_id":"1","name":"RELION","version":"3.0"},{"category":"CLASSIFICATION","id":"12","image_processing_id":"1","name":"RELION","version":"3.0"},{"category":"RECONSTRUCTION","id":"13","image_processing_id":"1","name":"RELION","version":"3.0"}],"em_specimen":[{"embedding_applied":"NO","experiment_id":"1","id":"1","shadowing_applied":"NO","staining_applied":"NO","vitrification_applied":"YES"}],"em_vitrification":[{"chamber_temperature":277.15,"cryogen_name":"ETHANE","details":"3 microL of sample applied to negatively glow discharged grid, blot force -10; blotting time 8.5 sec","humidity":92.0,"id":"1","instrument":"FEI VITROBOT MARK IV","specimen_id":"1"}],"entry":{"id":"6HU9"},"exptl":[{"method":"ELECTRON MICROSCOPY"}],"pdbx_audit_revision_category":[{"category":"citation","data_content_type":"Structure model","ordinal":1,"revision_ordinal":2},{"category":"citation_author","data_content_type":"Structure model","ordinal":2,"revision_ordinal":2},{"category":"pdbx_database_proc","data_content_type":"Structure model","ordinal":3,"revision_ordinal":2},{"category":"citation","data_content_type":"Structure model","ordinal":4,"revision_ordinal":3},{"category":"citation_author","data_content_type":"Structure model","ordinal":5,"revision_ordinal":3},{"category":"pdbx_database_proc","data_content_type":"Structure model","ordinal":6,"revision_ordinal":3},{"category":"pdbx_validate_close_contact","data_content_type":"Structure model","ordinal":7,"revision_ordinal":4},{"category":"struct_conn","data_content_type":"Structure model","ordinal":8,"revision_ordinal":4},{"category":"struct_site","data_content_type":"Structure model","ordinal":9,"revision_ordinal":4},{"category":"struct_site_gen","data_content_type":"Structure model","ordinal":10,"revision_ordinal":4},{"category":"atom_sites","data_content_type":"Structure model","ordinal":11,"revision_ordinal":5},{"category":"cell","data_content_type":"Structure model","ordinal":12,"revision_ordinal":5},{"category":"chem_comp_atom","data_content_type":"Structure model","ordinal":13,"revision_ordinal":6},{"category":"chem_comp_bond","data_content_type":"Structure model","ordinal":14,"revision_ordinal":6},{"category":"database_2","data_content_type":"Structure model","ordinal":15,"revision_ordinal":6},{"category":"em_3d_fitting_list","data_content_type":"Structure model","ordinal":16,"revision_ordinal":6},{"category":"pdbx_entry_details","data_content_type":"Structure model","ordinal":17,"revision_ordinal":6},{"category":"pdbx_initial_refinement_model","data_content_type":"Structure model","ordinal":18,"revision_ordinal":6},{"category":"pdbx_modification_feature","data_content_type":"Structure model","ordinal":19,"revision_ordinal":6},{"category":"pdbx_validate_chiral","data_content_type":"Structure model","ordinal":20,"revision_ordinal":6},{"category":"struct_conn","data_content_type":"Structure model","ordinal":21,"revision_ordinal":6},{"category":"struct_conn_type","data_content_type":"Structure model","ordinal":22,"revision_ordinal":6}],"pdbx_audit_revision_details":[{"data_content_type":"Structure model","ordinal":1,"provider":"repository","revision_ordinal":1,"type":"Initial release"}],"pdbx_audit_revision_group":[{"data_content_type":"Structure model","group":"Data collection","ordinal":1,"revision_ordinal":2},{"data_content_type":"Structure model","group":"Database references","ordinal":2,"revision_ordinal":2},{"data_content_type":"Structure model","group":"Data collection","ordinal":3,"revision_ordinal":3},{"data_content_type":"Structure model","group":"Database references","ordinal":4,"revision_ordinal":3},{"data_content_type":"Structure model","group":"Advisory","ordinal":5,"revision_ordinal":4},{"data_content_type":"Structure model","group":"Data collection","ordinal":6,"revision_ordinal":4},{"data_content_type":"Structure model","group":"Derived calculations","ordinal":7,"revision_ordinal":4},{"data_content_type":"Structure model","group":"Other","ordinal":8,"revision_ordinal":5},{"data_content_type":"Structure model","group":"Data collection","ordinal":9,"revision_ordinal":6},{"data_content_type":"Structure model","group":"Database references","ordinal":10,"revision_ordinal":6},{"data_content_type":"Structure model","group":"Derived calculations","ordinal":11,"revision_ordinal":6},{"data_content_type":"Structure model","group":"Refinement description","ordinal":12,"revision_ordinal":6},{"data_content_type":"Structure model","group":"Structure summary","ordinal":13,"revision_ordinal":6}],"pdbx_audit_revision_history":[{"data_content_type":"Structure model","major_revision":1,"minor_revision":0,"ordinal":1,"revision_date":"2018-12-26T00:00:00.000+00:00"},{"data_content_type":"Structure model","major_revision":1,"minor_revision":1,"ordinal":2,"revision_date":"2019-01-09T00:00:00.000+00:00"},{"data_content_type":"Structure model","major_revision":1,"minor_revision":2,"ordinal":3,"revision_date":"2019-01-16T00:00:00.000+00:00"},{"data_content_type":"Structure model","major_revision":1,"minor_revision":3,"ordinal":4,"revision_date":"2019-01-23T00:00:00.000+00:00"},{"data_content_type":"Structure model","major_revision":1,"minor_revision":4,"ordinal":5,"revision_date":"2019-12-11T00:00:00.000+00:00"},{"data_content_type":"Structure model","major_revision":1,"minor_revision":5,"ordinal":6,"revision_date":"2024-11-20T00:00:00.000+00:00"}],"pdbx_audit_revision_item":[{"data_content_type":"Structure model","item":"_citation.journal_abbrev","ordinal":1,"revision_ordinal":2},{"data_content_type":"Structure model","item":"_citation.pdbx_database_id_PubMed","ordinal":2,"revision_ordinal":2},{"data_content_type":"Structure model","item":"_citation.title","ordinal":3,"revision_ordinal":2},{"data_content_type":"Structure model","item":"_citation_author.identifier_ORCID","ordinal":4,"revision_ordinal":2},{"data_content_type":"Structure model","item":"_citation_author.name","ordinal":5,"revision_ordinal":2},{"data_content_type":"Structure model","item":"_citation.journal_volume","ordinal":6,"revision_ordinal":3},{"data_content_type":"Structure model","item":"_citation.page_first","ordinal":7,"revision_ordinal":3},{"data_content_type":"Structure model","item":"_citation.page_last","ordinal":8,"revision_ordinal":3},{"data_content_type":"Structure model","item":"_citation.year","ordinal":9,"revision_ordinal":3},{"data_content_type":"Structure model","item":"_citation_author.identifier_ORCID","ordinal":10,"revision_ordinal":3},{"data_content_type":"Structure model","item":"_atom_sites.fract_transf_matrix[1][1]","ordinal":11,"revision_ordinal":5},{"data_content_type":"Structure model","item":"_atom_sites.fract_transf_matrix[2][2]","ordinal":12,"revision_ordinal":5},{"data_content_type":"Structure model","item":"_atom_sites.fract_transf_matrix[3][3]","ordinal":13,"revision_ordinal":5},{"data_content_type":"Structure model","item":"_cell.Z_PDB","ordinal":14,"revision_ordinal":5},{"data_content_type":"Structure model","item":"_database_2.pdbx_DOI","ordinal":15,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_database_2.pdbx_database_accession","ordinal":16,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_em_3d_fitting_list.accession_code","ordinal":17,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_em_3d_fitting_list.initial_refinement_model_id","ordinal":18,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_em_3d_fitting_list.source_name","ordinal":19,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_em_3d_fitting_list.type","ordinal":20,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.conn_type_id","ordinal":21,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.id","ordinal":22,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.pdbx_dist_value","ordinal":23,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.pdbx_leaving_atom_flag","ordinal":24,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr1_auth_asym_id","ordinal":25,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr1_auth_comp_id","ordinal":26,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr1_auth_seq_id","ordinal":27,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr1_label_asym_id","ordinal":28,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr1_label_atom_id","ordinal":29,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr1_label_comp_id","ordinal":30,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr1_label_seq_id","ordinal":31,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr2_auth_asym_id","ordinal":32,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr2_auth_comp_id","ordinal":33,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr2_auth_seq_id","ordinal":34,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr2_label_asym_id","ordinal":35,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr2_label_atom_id","ordinal":36,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr2_label_comp_id","ordinal":37,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn.ptnr2_label_seq_id","ordinal":38,"revision_ordinal":6},{"data_content_type":"Structure model","item":"_struct_conn_type.id","ordinal":39,"revision_ordinal":6}],"pdbx_audit_support":[{"country":"United Kingdom","funding_organization":"Medical Research Council (United Kingdom)","grant_number":"MR/M00936X/1","ordinal":1},{"country":"United Kingdom","funding_organization":"Wellcome Trust","grant_number":"105628/Z/14/Z","ordinal":2}],"pdbx_database_related":[{"content_type":"associated EM volume","db_id":"EMD-0262","db_name":"EMDB","details":"BC1 (CHAINS A,B,C,D,E,F,G,H,I,J,L,M,N,O,P,Q,R,S,T,U)"},{"content_type":"other EM volume","db_id":"EMD-0269","db_name":"EMDB","details":"CIV 1 (chains a,b,c,d,e,f,g,h,i,j,k,l,y)"},{"content_type":"other EM volume","db_id":"EMD-0268","db_name":"EMDB","details":"CIV 2 (chains m,n,o,p,q,r,s,t,u,v,w,z)"}],"pdbx_database_status":{"SG_entry":"N","deposit_site":"PDBE","pdb_format_compatible":"Y","process_site":"PDBE","recvd_initial_deposition_date":"2018-10-05T00:00:00.000+00:00","status_code":"REL"},"pdbx_initial_refinement_model":[{"accession_code":"1KYO","id":1,"source_name":"PDB","type":"experimental model"},{"accession_code":"1V54","id":2,"source_name":"PDB","type":"experimental model"}],"pdbx_vrpt_summary":{"attempted_validation_steps":"visualanalysis,mogul,buster-report,molprobity,validation-pack,validation_schema,percentiles,writexml,writecif,writepdf","ligands_for_buster_report":"Y","report_creation_date":"2026-03-05T21:53:00.000+00:00"},"pdbx_vrpt_summary_em":[{"Q_score":0.431,"atom_inclusion_all_atoms":0.76,"atom_inclusion_backbone":0.817,"author_provided_fsc_resolution_by_cutoff_halfbit":3.32,"author_provided_fsc_resolution_by_cutoff_onebit":3.55,"author_provided_fsc_resolution_by_cutoff_pt_143":3.28,"author_provided_fsc_resolution_by_cutoff_pt_333":3.54,"author_provided_fsc_resolution_by_cutoff_pt_5":3.79,"contour_level_primary_map":0.0263,"exp_method":"electron microscopy"}],"pdbx_vrpt_summary_geometry":[{"angles_RMSZ":0.55,"bonds_RMSZ":0.36,"clashscore":5.7,"num_H_reduce":63043,"num_angles_RMSZ":84548,"num_bonds_RMSZ":62220,"percent_ramachandran_outliers":0.04,"percent_rotamer_outliers":8.49}],"rcsb_accession_info":{"deposit_date":"2018-10-05T00:00:00.000+00:00","has_released_experimental_data":"Y","initial_release_date":"2018-12-26T00:00:00.000+00:00","major_revision":1,"minor_revision":5,"revision_date":"2024-11-20T00:00:00.000+00:00","status_code":"REL"},"rcsb_entry_container_identifiers":{"assembly_ids":["1"],"emdb_ids":["EMD-0262"],"entity_ids":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35"],"entry_id":"6HU9","model_ids":[1],"non_polymer_entity_ids":["23","24","25","26","27","28","29","30","31","32","33","34","35"],"polymer_entity_ids":["1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22"],"pubmed_id":30598554,"rcsb_id":"6HU9","related_emdb_ids":["EMD-0269","EMD-0268"]},"rcsb_entry_info":{"assembly_count":1,"branched_entity_count":0,"cis_peptide_count":4,"deposited_atom_count":63031,"deposited_deuterated_water_count":0,"deposited_hydrogen_atom_count":0,"deposited_model_count":1,"deposited_modeled_polymer_monomer_count":7636,"deposited_nonpolymer_entity_instance_count":69,"deposited_polymer_entity_instance_count":44,"deposited_polymer_monomer_count":7922,"deposited_solvent_atom_count":0,"deposited_unmodeled_polymer_monomer_count":286,"disulfide_bond_count":10,"entity_count":35,"experimental_method":"EM","experimental_method_count":1,"inter_mol_covalent_bond_count":4,"inter_mol_metalic_bond_count":70,"molecular_weight":936.43,"na_polymer_entity_types":"Other","nonpolymer_bound_components":["CA","CU","CUA","FES","HEA","HEC","HEM","MG","ZN"],"nonpolymer_entity_count":13,"nonpolymer_molecular_weight_maximum":1.46,"nonpolymer_molecular_weight_minimum":0.02,"polymer_composition":"heteromeric protein","polymer_entity_count":22,"polymer_entity_count_DNA":0,"polymer_entity_count_RNA":0,"polymer_entity_count_nucleic_acid":0,"polymer_entity_count_nucleic_acid_hybrid":0,"polymer_entity_count_protein":22,"polymer_entity_taxonomy_count":22,"polymer_molecular_weight_maximum":58.83,"polymer_molecular_weight_minimum":5.38,"polymer_monomer_count_maximum":534,"polymer_monomer_count_minimum":47,"resolution_combined":[3.35],"selected_polymer_entity_types":"Protein (only)","software_programs_combined":["CTFFIND","EPU","GAUTOMATCH","PHENIX","RELION","UCSF CHIMERA"],"solvent_entity_count":0,"structure_determination_methodology":"experimental","structure_determination_methodology_priority":10},"rcsb_external_references":[{"id":"EMD-0262","link":"https://www.emdataresource.org/EMD-0262","type":"EM DATA RESOURCE"}],"rcsb_primary_citation":{"country":"US","id":"primary","journal_abbrev":"Nat. Struct. Mol. Biol.","journal_id_ISSN":"1545-9985","journal_volume":"26","page_first":"78","page_last":"83","pdbx_database_id_DOI":"10.1038/s41594-018-0172-z","pdbx_database_id_PubMed":30598554,"rcsb_ORCID_identifiers":["?","?","?","?","?","?","?","?"],"rcsb_authors":["Hartley, A.M.","Lukoyanova, N.","Zhang, Y.","Cabrera-Orefice, A.","Arnold, S.","Meunier, B.","Pinotsis, N.","Marechal, A."],"rcsb_journal_abbrev":"Nat Struct Mol Biol","title":"Structure of yeast cytochrome c oxidase in a supercomplex with cytochrome bc1.","year":2019},"refine_ls_restr":[{"dev_ideal":0.008,"number":64705,"pdbx_refine_id":"ELECTRON MICROSCOPY","type":"f_bond_d"},{"dev_ideal":1.069,"number":87763,"pdbx_refine_id":"ELECTRON MICROSCOPY","type":"f_angle_d"},{"dev_ideal":17.856,"number":38221,"pdbx_refine_id":"ELECTRON MICROSCOPY","type":"f_dihedral_angle_d"},{"dev_ideal":0.057,"number":9460,"pdbx_refine_id":"ELECTRON MICROSCOPY","type":"f_chiral_restr"},{"dev_ideal":0.008,"number":10892,"pdbx_refine_id":"ELECTRON MICROSCOPY","type":"f_plane_restr"}],"struct":{"pdbx_CASP_flag":"N","title":"III2-IV2 mitochondrial respiratory supercomplex from S. cerevisiae"},"struct_keywords":{"pdbx_keywords":"OXIDOREDUCTASE/ELECTRON TRANSPORT","text":"Cytochrome c oxidase Cytochrome bc1 Mitochondria Respiratory chain Supercomplex, OXIDOREDUCTASE, ELECTRON TRANSPORT, OXIDOREDUCTASE-ELECTRON TRANSPORT complex"},"rcsb_id":"6HU9"} \ No newline at end of file