Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,17 @@ BioJava 7.3.0
* `LocalPDBDirectory.getMiddleHash(String)`, computing the two-character directory from the
right so that it is correct for both short and extended PDB IDs #1133
* Support for the ECOD distribution format introduced at v294.1 #1141 #1139
* `StructureInterfaceList.clusterInterfaces()`, which clusters interfaces starting from any
given clusters and with a custom identifier pair for matching (e.g. `ENTITY_ID_PAIR`)

### Performance
* Contact calculation is about 1.5x faster: squared distances compared against a squared cutoff,
primitive arrays in `GridCell` instead of boxed lists, and a pre-sized `AtomContactSet` #1147
* ASA calculation is about 1.2-1.3x faster, by the same replacement of objects with flat
primitive arrays in the hot loops #1148
* Interface clustering in `StructureInterfaceList.getClusters()` uses a greedy leader algorithm
instead of single linkage, needing at most n(n-1)/2 contact overlap scores and usually far fewer.
For 1gav (690 interfaces) it takes 45 ms instead of 8.3 s
* `EcodInstallation.getVersion()` reads the file header instead of parsing every domain. The
current release is 653 MB and holds nearly three million records #1141

Expand All @@ -43,6 +48,10 @@ BioJava 7.3.0
### Changed
* `createValidationFiles()` now defaults to `ETagPolicy.USE_IF_HEX_DIGEST`, so existing callers
begin recording checksums where the server offers one #1133
* `StructureInterfaceList.getClusters()`: results can depend on the order of the interfaces (by
default descending by area), members are no longer sorted by id, and
`StructureInterfaceCluster.getAverageScore()` is now the average score of the members merged into
the representative rather than over all pairs of members
* Integration tests run nightly rather than on every pull request #1137 #1135
* Tests migrated to JUnit 5 #1125 #1126 #1038
* Library upgrades #1130 #1132
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
/*
* BioJava development code
*
* This code may be freely distributed and modified under the
* terms of the GNU Lesser General Public Licence. This should
* be distributed with the code. If you do not have a copy,
* see:
*
* http://www.gnu.org/copyleft/lesser.html
*
* Copyright for this code is held jointly by the individual
* authors. These should be listed in @author doc comments.
*
* For more information on the BioJava project and its aims,
* or to join the biojava-l mailing list, visit the home page
* at:
*
* http://www.biojava.org/
*
*/
package org.biojava.nbio.structure.test.contact;

import org.biojava.nbio.core.util.SingleLinkageClusterer;
import org.biojava.nbio.structure.Structure;
import org.biojava.nbio.structure.StructureException;
import org.biojava.nbio.structure.StructureIO;
import org.biojava.nbio.structure.align.util.AtomCache;
import org.biojava.nbio.structure.contact.InterfaceFinder;
import org.biojava.nbio.structure.contact.StructureInterface;
import org.biojava.nbio.structure.contact.StructureInterfaceCluster;
import org.biojava.nbio.structure.contact.StructureInterfaceList;
import org.biojava.nbio.structure.io.FileParsingParameters;
import org.biojava.nbio.structure.io.StructureFiletype;
import org.junit.Ignore;
import org.junit.Test;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static org.junit.Assert.assertEquals;

/**
* Compares the performance of interface clustering with {@link StructureInterfaceList#clusterInterfaces}
* (leader algorithm) against single linkage clustering with {@link SingleLinkageClusterer}, for an assembly with many interfaces.
*
* By default it is ignored.
* To execute use:
* <pre>
* mvn -Dtest=TestInterfaceClusteringPerformance test
* </pre>
*/
public class TestInterfaceClusteringPerformance {

/** An icosahedral capsid with 180 chains of one entity and ~700 interfaces. Larger cases like 5vf3 or 3j3q take very long */
private static final String PDB_ID = "1gav";

@Ignore("Performance test to be run manually")
@Test
public void testClusteringPerformance() throws IOException, StructureException {
AtomCache cache = new AtomCache();
FileParsingParameters params = new FileParsingParameters();
params.setAlignSeqRes(true);
cache.setFileParsingParams(params);
cache.setFiletype(StructureFiletype.CIF);
StructureIO.setAtomCache(cache);

Structure assembly = StructureIO.getBiologicalAssembly(PDB_ID, 1, false);
List<StructureInterface> list = new InterfaceFinder(assembly).getAllInterfaces().getList();
System.out.printf("Found %d interfaces in assembly 1 of %s%n", list.size(), PDB_ID);

double cutoff = StructureInterfaceList.DEFAULT_CONTACT_OVERLAP_SCORE_CLUSTER_CUTOFF;

// first calculations of contact overlap scores initialise the contact sets (lazily): we do that before timing
for (StructureInterface interf : list) {
interf.getGroupContacts();
}

long start = System.currentTimeMillis();
List<StructureInterfaceCluster> singletons = new ArrayList<>();
for (StructureInterface interf : list) {
StructureInterfaceCluster cluster = new StructureInterfaceCluster();
cluster.addMember(interf);
singletons.add(cluster);
}
List<StructureInterfaceCluster> leaderClusters = StructureInterfaceList.clusterInterfaces(singletons, StructureInterfaceList.ENTITY_ID_PAIR, cutoff);
long end = System.currentTimeMillis();
System.out.printf("%d clusters found. Time for leader clustering: %d ms%n", leaderClusters.size(), end - start);

start = System.currentTimeMillis();
double[][] matrix = new double[list.size()][list.size()];
for (int i = 0; i < list.size(); i++) {
for (int j = i + 1; j < list.size(); j++) {
matrix[i][j] = Math.max(list.get(i).getContactOverlapScore(list.get(j), false), list.get(i).getContactOverlapScore(list.get(j), true));
}
}
long endMatrix = System.currentTimeMillis();
Map<Integer, Set<Integer>> singleLinkageClusters = new SingleLinkageClusterer(matrix, true).getClusters(cutoff);
end = System.currentTimeMillis();
System.out.printf("%d clusters found. Time for single linkage clustering: %d ms (all-vs-all scores %d ms, clustering %d ms)%n",
singleLinkageClusters.size(), end - start, endMatrix - start, end - endMatrix);

assertEquals(list.size(), leaderClusters.stream().mapToInt(c -> c.getMembers().size()).sum());
assertEquals(list.size(), singleLinkageClusters.values().stream().mapToInt(Set::size).sum());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -651,7 +651,7 @@ public double getContactOverlapScore(StructureInterface other, boolean invert) {
}

/**
* This method check if two compounds have same MolIds or not.
* Checks whether the 2 pairs of entities (compounds) are composed by the same entity IDs (molId).
* @param thisCompounds
* @param otherCompounds
* @return
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,12 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Function;

import org.biojava.nbio.core.util.SingleLinkageClusterer;
import org.biojava.nbio.structure.Atom;
import org.biojava.nbio.structure.Chain;
import org.biojava.nbio.structure.EntityInfo;
import org.biojava.nbio.structure.Structure;
import org.biojava.nbio.structure.asa.AsaCalculator;
import org.biojava.nbio.structure.xtal.CrystalBuilder;
Expand Down Expand Up @@ -368,6 +369,10 @@ public List<StructureInterfaceCluster> getClusters() {
* using Jaccard contact set scores to measure the similarity of interfaces.
* Subsequent calls will use the cached value without recomputing the clusters.
* The clusters will be assigned ids by sorting descending by {@link StructureInterfaceCluster#getTotalArea()}
* <p>
* Interfaces are compared by entity pairs (see {@link #ENTITY_ID_PAIR}) and grouped with the leader
* algorithm described in {@link #clusterInterfaces(List, Function, double)}, thus the result depends on
* the order of the list (by default descending by area, see {@link #sort()}).
* @param contactOverlapScoreClusterCutoff the contact overlap score above which a pair will be
* clustered
* @return
Expand All @@ -377,77 +382,125 @@ public List<StructureInterfaceCluster> getClusters(double contactOverlapScoreClu
return clusters;
}

clusters = new ArrayList<>();

// nothing to do if we have no interfaces
if (list.isEmpty()) return clusters;

logger.debug("Calculating all-vs-all Jaccard scores for {} interfaces", list.size());
double[][] matrix = new double[list.size()][list.size()];
List<StructureInterfaceCluster> singletons = new ArrayList<>(list.size());
for (StructureInterface interf : list) {
StructureInterfaceCluster cluster = new StructureInterfaceCluster();
cluster.addMember(interf);
singletons.add(cluster);
}

for (int i=0;i<list.size();i++) {
for (int j=i+1;j<list.size();j++) {
StructureInterface iInterf = list.get(i);
StructureInterface jInterf = list.get(j);
clusters = clusterInterfaces(singletons, ENTITY_ID_PAIR, contactOverlapScoreClusterCutoff);

double scoreDirect = iInterf.getContactOverlapScore(jInterf, false);
double scoreInvert = iInterf.getContactOverlapScore(jInterf, true);
// now we sort by areas (descending) and assign ids based on that sorting
clusters.sort((o1, o2) -> Double.compare(o2.getTotalArea(), o1.getTotalArea())); //note we invert so that sorting is descending

double maxScore = Math.max(scoreDirect, scoreInvert);
int id = 1;
for (StructureInterfaceCluster cluster:clusters) {
cluster.setId(id);
id++;
}

matrix[i][j] = maxScore;
}
return clusters;
}

/**
* Identifies each side of an interface by the entity id ({@link EntityInfo#getMolId()}) of its parent chain.
* Returns null if any of the parent chains or entities can't be found.
* @see #clusterInterfaces(List, Function, double)
*/
public static final Function<StructureInterface, Pair<Integer>> ENTITY_ID_PAIR = interf -> {
Pair<Chain> chains = interf.getParentChains();
if (chains == null || chains.getFirst().getEntityInfo() == null || chains.getSecond().getEntityInfo() == null) {
return null;
}
return new Pair<>(chains.getFirst().getEntityInfo().getMolId(), chains.getSecond().getEntityInfo().getMolId());
};

logger.debug("Will now cluster {} interfaces based on full all-vs-all Jaccard scores matrix", list.size());
SingleLinkageClusterer slc = new SingleLinkageClusterer(matrix, true);
/**
* Groups interface clusters further, merging any two whose representatives (first members) have a
* contact overlap score above the given cutoff.
* <p>
* This is the leader algorithm: the representative of the first cluster is compared to the representatives of all
* subsequent clusters, merging those above cutoff into it; then the same is repeated for the next remaining cluster.
* It performs at most n(n-1)/2 score calculations, and about n*k for n input and k output clusters.
* Unlike single linkage clustering, similarity is not transitive through non-representative members
* and the result depends on the input order: placing larger interfaces first makes them the representatives.
* <p>
* Two interfaces are only compared if their identifier pairs match, in the same or inverted order.
* The score is then {@link StructureInterface#getContactOverlapScore(StructureInterface, boolean)} non-inverted
* or inverted respectively, or the maximum of both if all 4 identifiers are equal.
* <p>
* The input clusters are not modified. The output clusters are new objects whose members are the
* representative's cluster members followed by the members of the clusters merged into it, and whose average
* score is the average of the scores that caused merges (1.0 if no merges). The cluster back-references of all
* members are set to the output clusters. Ids are not assigned.
* @param initialClusters the starting clusters, e.g. one singleton cluster per interface. The first member of
* each is its representative
* @param idPairFunction gives the pair of identifiers for the 2 sides of an interface, e.g. {@link #ENTITY_ID_PAIR}.
* If it returns null the interface is not merged with any other
* @param contactOverlapScoreClusterCutoff the contact overlap score above which a pair of clusters is merged
* @return the merged clusters, in the order of their representatives in the input
* @since 7.3.0
*/
public static <T> List<StructureInterfaceCluster> clusterInterfaces(List<StructureInterfaceCluster> initialClusters,
Function<StructureInterface, Pair<T>> idPairFunction,
double contactOverlapScoreClusterCutoff) {
int n = initialClusters.size();
logger.debug("Clustering {} interface clusters by contact overlap score", n);

List<StructureInterface> reps = new ArrayList<>(n);
List<Pair<T>> idPairs = new ArrayList<>(n);
for (StructureInterfaceCluster cluster : initialClusters) {
StructureInterface rep = cluster.getMembers().get(0);
reps.add(rep);
idPairs.add(idPairFunction.apply(rep));
}

Map<Integer, Set<Integer>> clusteredIndices = slc.getClusters(contactOverlapScoreClusterCutoff);
for (int clusterIdx:clusteredIndices.keySet()) {
List<StructureInterface> members = new ArrayList<>();
for (int idx:clusteredIndices.get(clusterIdx)) {
members.add(list.get(idx));
}
List<StructureInterfaceCluster> result = new ArrayList<>();
boolean[] merged = new boolean[n];
for (int i = 0; i < n; i++) {
if (merged[i]) continue;
StructureInterfaceCluster cluster = new StructureInterfaceCluster();
cluster.setMembers(members);
double averageScore = 0.0;
int countPairs = 0;
for (int i=0;i<members.size();i++) {
int iIdx = list.indexOf(members.get(i));
for (int j=i+1;j<members.size();j++) {
averageScore += matrix[iIdx][list.indexOf(members.get(j))];
countPairs++;
cluster.getMembers().addAll(initialClusters.get(i).getMembers());
double sumScores = 0;
int countMerges = 0;
// descending, so that merged members are appended in the same order as in previous implementations
for (int j = n - 1; j > i; j--) {
if (merged[j]) continue;
double score = getContactOverlapScore(reps.get(i), idPairs.get(i), reps.get(j), idPairs.get(j));
if (score > contactOverlapScoreClusterCutoff) {
cluster.getMembers().addAll(initialClusters.get(j).getMembers());
merged[j] = true;
sumScores += score;
countMerges++;
}
}
if (countPairs>0) {
averageScore = averageScore/countPairs;
} else {
// if only one interface in cluster we set the score to the maximum
averageScore = 1.0;
cluster.setAverageScore(countMerges > 0 ? sumScores / countMerges : 1.0);
for (StructureInterface member : cluster.getMembers()) {
member.setCluster(cluster);
}
cluster.setAverageScore(averageScore);
clusters.add(cluster);
result.add(cluster);
}

// finally we have to set the back-references in each StructureInterface
for (StructureInterfaceCluster cluster:clusters) {
for (StructureInterface interf:cluster.getMembers()) {
interf.setCluster(cluster);
}
}
logger.debug("Done clustering {} interfaces based on full all-vs-all Jaccard scores matrix. Found a total of {} clusters", list.size(), clusters.size());

// now we sort by areas (descending) and assign ids based on that sorting
clusters.sort((o1, o2) -> Double.compare(o2.getTotalArea(), o1.getTotalArea())); //note we invert so that sorting is descending
logger.debug("Found {} clusters from {} input clusters at contact overlap score cutoff {}", result.size(), n, contactOverlapScoreClusterCutoff);
return result;
}

int id = 1;
for (StructureInterfaceCluster cluster:clusters) {
cluster.setId(id);
id++;
private static <T> double getContactOverlapScore(StructureInterface iInterf, Pair<T> iIds, StructureInterface jInterf, Pair<T> jIds) {
if (iIds == null || jIds == null) {
return 0;
}

return clusters;
boolean direct = iIds.getFirst().equals(jIds.getFirst()) && iIds.getSecond().equals(jIds.getSecond());
boolean inverted = iIds.getFirst().equals(jIds.getSecond()) && iIds.getSecond().equals(jIds.getFirst());
if (direct && inverted) {
// all 4 ids are the same: the order of the sides is not known, so we try both
return Math.max(iInterf.getContactOverlapScore(jInterf, false), iInterf.getContactOverlapScore(jInterf, true));
} else if (direct) {
return iInterf.getContactOverlapScore(jInterf, false);
} else if (inverted) {
return iInterf.getContactOverlapScore(jInterf, true);
}
return 0;
}

@Override
Expand Down
Loading
Loading