Git Lab CI for docker build enabled! You can enable it using .gitlab-ci.yml in your project. Check file template at https://gitlab.bio.di.uminho.pt/snippets/5

Commit 3305d1d3 authored by Davide Lagoa's avatar Davide Lagoa
Browse files

remote service to neo4j database created

parent 62ca0537
......@@ -10,15 +10,15 @@
</description>
<repositories>
<repository>
<id>nexus</id>
<name>Nexus BioSystems Server</name>
<!-- <url>http://192.168.1.99/nexus/content/groups/public/</url> -->
<!-- <url>http://193.137.11.210/nexus/content/groups/public/</url> -->
<url>https://www.bio.di.uminho.pt/nexus/content/groups/public/</url>
</repository>
</repository>
<repository>
<id>ebi-repo</id>
......@@ -95,6 +95,16 @@
</build>
<dependencies>
<!-- <dependency> <groupId>org.neo4j.driver</groupId> <artifactId>neo4j-java-driver</artifactId>
<version>1.7.2</version> </dependency> -->
<!-- <dependency> <groupId>org.neo4j</groupId> <artifactId>neo4j-jdbc-bolt</artifactId>
<version>3.4.0</version> </dependency> -->
<dependency>
<groupId>org.neo4j.driver</groupId>
<artifactId>neo4j-java-driver</artifactId>
<version>1.7.2</version>
</dependency>
<dependency>
<groupId>pt.uminho.ceb.biosystems.transyt</groupId>
<artifactId>utilities</artifactId>
......
......@@ -37,10 +37,13 @@ import pt.uminho.ceb.biosystems.transyt.service.internalDB.WriteByMetabolitesID;
import pt.uminho.ceb.biosystems.transyt.service.kbase.ModelSEEDCompoundsFileReader;
import pt.uminho.ceb.biosystems.transyt.service.kbase.Reports;
import pt.uminho.ceb.biosystems.transyt.service.kbase.Tools;
import pt.uminho.ceb.biosystems.transyt.service.neo4jRest.RestNeo4jGraphDatabase;
import pt.uminho.ceb.biosystems.transyt.service.relations.GPRAssociations;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.TransytGeneralProperties;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.TransytGraphDatabaseService;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.TransytNeo4jInitializer;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.TransytNode;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.TransytRelationship;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.TransytRelationshipType;
import pt.uminho.ceb.biosystems.transyt.utilities.biocomponents.OutputTransytFormat;
import pt.uminho.ceb.biosystems.transyt.utilities.biocomponents.TransytSBMLLevel3Writer;
......@@ -63,7 +66,7 @@ public class ProvideTransportReactionsToGenes {
// private List<String> list = List.of("cpd00137", "cpd10516", "cpd01914", "cpd08376", "cpd11581", "cpd11586", "cpd11588", "cpd11590", "cpd15603", "cpd15604", "cpd00009", "cpd11576", "cpd11582", "cpd11592", "cpd15606", "cpd00076", "cpd00122", "cpd00158", "cpd00179", "cpd00314", "cpd00492", "cpd00588", "cpd00794", "cpd01030", "cpd01171", "cpd03696", "cpd15561", "cpd15499", "cpd00079", "cpd00235", "cpd01912", "cpd00209", "cpd15499", "cpd00209", "cpd15561", "cpd11640", "cpd15352", "cpd11640", "cpd15500", "cpd11640", "cpd15560", "cpd00036", "cpd00041", "cpd00106", "cpd00130", "cpd00047", "cpd15560", "cpd00036", "cpd00041", "cpd00001", "cpd00027", "cpd15560", "cpd00024", "cpd00036", "cpd00137", "cpd00039", "cpd01155", "cpd00051", "cpd00064", "cpd00063", "cpd00067", "cpd00064", "cpd00118", "cpd00811", "cpd15353", "cpd00811", "cpd15499", "cpd00971", "cpd00080", "cpd00142", "cpd00162", "cpd00205", "cpd00305", "cpd00609", "cpd01015", "cpd00266", "cpd00870", "cpd15378");
private Map<String, List<AlignmentCapsule>> blastResults;
private TransytGraphDatabaseService service;
private RestNeo4jGraphDatabase service;
// private Map<String, Set<String>> homologousGenes;
private String[] taxonomy;
private String organism;
......@@ -132,13 +135,13 @@ public class ProvideTransportReactionsToGenes {
queryFileTotalOfGenes = blast.getQueryFileSize();
GraphDatabaseService graphDatabaseService = TransytNeo4jInitializer.getDatabaseService(true, properties);
service = new TransytGraphDatabaseService(graphDatabaseService);
RestNeo4jGraphDatabase service = TransytNeo4jInitializer.getDatabaseService(true, properties);
// service = new TransytGraphDatabaseService(graphDatabaseService);
logger.info("Beginning transaction with neo4j TranSyT database...");
Transaction dataTx = graphDatabaseService.beginTx();
// Transaction dataTx = graphDatabaseService.beginTx();
//////////////////////////
// @SuppressWarnings("resource")
// Scanner reader = new Scanner(System.in);
......@@ -163,13 +166,16 @@ public class ProvideTransportReactionsToGenes {
//
// }
/////////////////////////
dataTx.close();
service.shutdown();
graphDatabaseService.shutdown();
// dataTx.close();
// service.shutdown();
// graphDatabaseService.shutdown();
service.close();
logger.info("Transaction terminated!");
logger.info("TranSyT neo4j database shutdown...");
logger.info("TranSyT shutdown!");
}
catch (Exception e) {
e.printStackTrace();
......@@ -944,7 +950,7 @@ public class ProvideTransportReactionsToGenes {
if(!tcNumbersNotPresentInTRIAGEdatabase.contains(tcNumber)) {
Node tcNumberNode = service.findTcNumberNode(tcNumber);
TransytNode tcNumberNode = service.findTcNumberNode(tcNumber);
// System.out.println(tcNumber);
// System.out.println(tcNumberNode == null);
......@@ -953,24 +959,24 @@ public class ProvideTransportReactionsToGenes {
Map<String, Boolean> reversibility = new HashMap<>();
if(!mainReactions.containsKey(tcNumber) && tcNumberNode.hasProperty(TransytGeneralProperties.MainReactionsIDs.toString())) {
if(!mainReactions.containsKey(tcNumber) && tcNumberNode.hasProperty(TransytGeneralProperties.MainReactionsIDs)) {
String ids = (String) tcNumberNode.getProperty(TransytGeneralProperties.MainReactionsIDs.toString());
String ids = (String) tcNumberNode.getProperty(TransytGeneralProperties.MainReactionsIDs);
mainReactions.put(tcNumber, Arrays.asList(ids.split(", ")));
}
Set<Node> reactionNodes = new HashSet<>();
Set<TransytNode> reactionNodes = new HashSet<>();
Set<String> ids = new HashSet<>();
Iterable<Relationship> relations = tcNumberNode.getRelationships(TransytRelationshipType.has_reaction);
Set<TransytRelationship> relations = service.getDownstreamRelationships(tcNumberNode.getNodeID(), TransytRelationshipType.has_reaction);
for(Relationship rel : relations) {
for(TransytRelationship rel : relations) {
Node node = rel.getEndNode();
TransytNode node = rel.getOtherEndNode();
boolean rev = Boolean.valueOf(rel.getProperty(TransytGeneralProperties.Reversible.toString()).toString());
boolean rev = Boolean.valueOf(rel.getProperty(TransytGeneralProperties.Reversible));
String reactionID = node.getProperty(TransytGeneralProperties.ReactionID.toString()).toString();
String reactionID = node.getProperty(TransytGeneralProperties.ReactionID);
reversibility.put(reactionID, rev);
......@@ -1043,7 +1049,7 @@ public class ProvideTransportReactionsToGenes {
*/
private void getTaxonomy(String uniprotAccession){
Node node = service.findUniprotAccessionNode(uniprotAccession);
TransytNode node = service.findUniprotAccessionNode(uniprotAccession);
if(node != null) {
......@@ -1051,11 +1057,11 @@ public class ProvideTransportReactionsToGenes {
// System.out.println(node.getAllProperties());
String[] tax = node.getProperty(TransytGeneralProperties.Taxonomy.toString()).toString().replaceAll("\\[", "").replaceAll("\\]", "").split(", ");
String[] tax = node.getProperty(TransytGeneralProperties.Taxonomy).replaceAll("\\[", "").replaceAll("\\]", "").split(", ");
taxonomies.put(uniprotAccession, tax);
organisms.put(uniprotAccession, node.getProperty(TransytGeneralProperties.Organism.toString()).toString());
organisms.put(uniprotAccession, node.getProperty(TransytGeneralProperties.Organism));
}
......@@ -1310,14 +1316,14 @@ public class ProvideTransportReactionsToGenes {
* @param reversibility
* @return
*/
private Set<String> getAllNodesReactionsIDsForAnnotation(Set<Node> reactionNodes, Map<String, Boolean> reversibility){
private Set<String> getAllNodesReactionsIDsForAnnotation(Set<TransytNode> reactionNodes, Map<String, Boolean> reversibility){
Set<String> set = new HashSet<>();
try {
for(Node node : reactionNodes) {
for(TransytNode node : reactionNodes) {
String reactionID = node.getProperty(TransytGeneralProperties.ReactionID.toString()).toString();
String reactionID = node.getProperty(TransytGeneralProperties.ReactionID);
// System.out.println(reactionID);
......@@ -1335,9 +1341,11 @@ public class ProvideTransportReactionsToGenes {
// System.out.println(reactionsToIgnore.contains(reactionID));
// System.out.println(node.hasProperty(defaultLabel.toString()));
// System.out.println(reactionID);
// System.out.println(reactionID);
if(!reactionsToIgnore.contains(reactionIDaux) && node.hasRelationship(defaultRelationshipReactants)) {
boolean go = service.nodeHasRelationshipType(node.getNodeID(), defaultRelationshipReactants);
if(!reactionsToIgnore.contains(reactionIDaux) && go) {
// System.out.println("entrou");
......@@ -1351,7 +1359,7 @@ public class ProvideTransportReactionsToGenes {
// System.out.println(node.getAllProperties());
// String reaction = node.getProperty(defaultLabel.toString()).toString();
String reaction = service.getReactionEquation(node, defaultRelationshipReactants, defaultRelationshipProducts,
String reaction = service.getReactionEquation(node.getNodeID(), defaultRelationshipReactants, defaultRelationshipProducts,
reversibility.get(reactionID));
metabolitesNames.putAll(service.getTemporaryCompoundsNames());
......@@ -1384,7 +1392,7 @@ public class ProvideTransportReactionsToGenes {
if(!reactionContainersByID.containsKey(reactionIDaux)) {
String reactionWithNames = node.getProperty(TransytGeneralProperties.Reaction.toString()).toString();
String reactionWithNames = node.getProperty(TransytGeneralProperties.Reaction);
ReactionContainer container = new ReactionContainer(reaction, reversibility.get(reactionID));
container.setReactionWithIDs(reactionWithNames);
......
......@@ -7,15 +7,16 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.neo4j.graphdb.Relationship;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import pt.uminho.ceb.biosystems.merlin.utilities.Pair;
import pt.uminho.ceb.biosystems.merlin.utilities.containers.capsules.AlignmentCapsule;
import pt.uminho.ceb.biosystems.transyt.service.internalDB.WriteByMetabolitesID;
import pt.uminho.ceb.biosystems.transyt.service.neo4jRest.RestNeo4jGraphDatabase;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.TransytGeneralProperties;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.TransytGraphDatabaseService;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.TransytNode;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.TransytRelationship;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.TransytRelationshipType;
import pt.uminho.ceb.biosystems.transyt.utilities.capsules.GeneContainer;
import pt.uminho.ceb.biosystems.transyt.utilities.capsules.ReactionContainer;
......@@ -282,7 +283,7 @@ public class GPRAssociations {
* @param reactionContainersByID
* @param finalResults
*/
public static Map<String, String> buildGeneRules(TransytGraphDatabaseService service, Map<String, Map<String, String>> proteinComplexes,
public static Map<String, String> buildGeneRules(RestNeo4jGraphDatabase service, Map<String, Map<String, String>> proteinComplexes,
Map<String, ReactionContainer> reactionContainersByID, Map<String, Map<String, Set<String>>> results) {
// Map<String, String> genes = FilesUtils.readMapFromFile("C:\\Users\\Davide\\OneDrive - Universidade do Minho\\UMinho\\Tese\\KBase\\Reports\\ecoli_Validation30\\geneRules.txt");
......@@ -345,14 +346,16 @@ public class GPRAssociations {
// Iterable<Relationship> relationships = service.findReactionNode(react.replace("CoA", "abc")).getRelationships(TransytRelationshipType.has_reaction);
Iterable<Relationship> relationships = service.findReactionNode(react.replace("iTR", "TR")).getRelationships(TransytRelationshipType.has_reaction);
TransytNode node = service.findReactionNode(react.replace("iTR", "TR"));
Set<TransytRelationship> relationships = service.getUpstreamRelationships(node.getNodeID(), TransytRelationshipType.has_reaction);
Set<String> TCs = new HashSet<>();
for(Relationship rel : relationships) {
for(TransytRelationship rel : relationships) {
// System.out.println(rel.getStartNode().getAllProperties());
TCs.add(rel.getStartNode().getProperty(TransytGeneralProperties.TC_Number.toString()).toString());
TCs.add(rel.getOtherEndNode().getProperty(TransytGeneralProperties.TC_Number));
}
......
......@@ -19,6 +19,7 @@ import org.neo4j.graphdb.Result;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.event.KernelEventHandler;
import org.neo4j.graphdb.event.TransactionEventHandler;
import org.neo4j.graphdb.index.IndexManager;
import org.neo4j.graphdb.schema.Schema;
import org.neo4j.graphdb.traversal.BidirectionalTraversalDescription;
import org.neo4j.graphdb.traversal.TraversalDescription;
......
package pt.uminho.ceb.biosystems.transyt.service.transytDatabase;
import java.io.File;
import org.neo4j.cypher.internal.ExecutionEngine;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.graphdb.security.URLAccessRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import pt.uminho.ceb.biosystems.transyt.service.internalDB.WriteByMetabolitesID;
import pt.uminho.ceb.biosystems.transyt.utilities.files.FilesUtils;
import pt.uminho.ceb.biosystems.transyt.service.neo4jRest.RestNeo4jGraphDatabase;
import pt.uminho.ceb.biosystems.transyt.utilities.transytUtilities.Properties;
public class TransytNeo4jInitializer {
private static final Logger logger = LoggerFactory.getLogger(WriteByMetabolitesID.class);
private static final String PATH = FilesUtils.getTranSyTDBDatabaseDirectory();
// private static final String PATH = FilesUtils.getTranSyTDBDatabaseDirectory();
/**
* initialize triage neo4j database.
*
* @return
*/
public static GraphDatabaseService getDatabaseService(boolean readOnly, Properties properties) {
logger.info("Initializing TranSyT graph database...");
public static RestNeo4jGraphDatabase getDatabaseService(boolean readOnly, Properties properties) {
logger.debug("TranSyT neo4j database location: {}", PATH.concat(properties.getTransytDBName()));
try {
String uri = properties.getDatabaseURI();
String username = properties.getDatabaseUsername();
String password = properties.getDatabasePassword();
logger.info("Initializing TranSyT graph database...");
GraphDatabaseService graphDb = null;
if(readOnly) {
logger.debug("Connecting to TranSyT neo4j database at: {}", properties.getDatabaseURI());
graphDb = new GraphDatabaseFactory()
.newEmbeddedDatabaseBuilder(new File(PATH.concat(properties.getTransytDBName())))
.setConfig( GraphDatabaseSettings.read_only, "true" )
.newGraphDatabase(); //faster
logger.info("TranSyT graph database (read-only) open!");
}
else {
graphDb = new GraphDatabaseFactory()
.newEmbeddedDatabase(new File(PATH.concat(properties.getTransytDBName())));
logger.info("TranSyT graph database open!");
return new RestNeo4jGraphDatabase(uri, username, password);
}
catch (Exception e) {
logger.error("Something went wrong while initializing connection to TranSyT's internal database. "
+ "TranSyT will shut down, please try again later!");
logger.trace("StackTrace {}",e);
System.exit(0);
}
return graphDb;
return null;
// if(readOnly) {
//
// graphDb = new GraphDatabaseFactory()
// .newEmbeddedDatabaseBuilder(new File(PATH.concat(properties.getTransytDBName())))
// .setConfig( GraphDatabaseSettings.read_only, "true" )
// .newGraphDatabase(); //faster
//
// logger.info("TranSyT graph database (read-only) open!");
// }
// else {
// graphDb = new GraphDatabaseFactory()
// .newEmbeddedDatabase(new File(PATH.concat(properties.getTransytDBName())));
//
// logger.info("TranSyT graph database open!");
// }
// return graphDb;
}
}
package pt.uminho.ceb.biosystems.transyt.service.transytDatabase;
import java.util.Map;
/**
* @author Davide
*
*/
public class TransytNode {
private Map<String, String> properties;
private int id;
public TransytNode(int id, Map<String, String> properties) {
this.id = id;
this.properties = properties;
}
/**
* Check if a node contains a property.
*
* @param property
* @return
*/
public boolean hasProperty(TransytGeneralProperties property) {
if(properties.containsKey(property.toString()))
return true;
return false;
}
/**
* Get a specific node property.
*
* @param property
* @return
*/
public String getProperty(TransytGeneralProperties property) {
try {
return properties.get(property.toString());
}
catch (Exception e) {
e.printStackTrace();
}
return null;
}
/**
* Get node id in the database
*
* @return
*/
public int getNodeID() {
return id;
}
}
package pt.uminho.ceb.biosystems.transyt.service.transytDatabase;
import java.util.Map;
public class TransytRelationship {
private int id;
private TransytNode node;
private Map<String, String> properties;
private TransytRelationshipType type;
public TransytRelationship(int id, Map<String, String> properties, TransytRelationshipType type,
TransytNode endNode){
this.id = id;
this.node = endNode;
this.properties = properties;
this.type = type;
}
/**
* @return the id
*/
public int getId() {
return id;
}
/**
* @return the node
*/
public TransytNode getOtherEndNode() {
return node;
}
/**
* @return the properties
*/
public Map<String, String> getProperties() {
return properties;
}
/**
* @return the type
*/
public TransytRelationshipType getType() {
return type;
}
/**
* Check if a relationship contains a property.
*
* @param property
* @return
*/
public boolean hasProperty(TransytGeneralProperties property) {
if(properties.containsKey(property.toString()))
return true;
return false;
}
/**
* Get a specific relationship property.
*
* @param property
* @return
*/
public String getProperty(TransytGeneralProperties property) {
try {
return properties.get(property.toString());
}
catch (Exception e) {
e.printStackTrace();
}
return null;
}
}
......@@ -104,42 +104,42 @@ class TriageMain {
System.out.println(compounds.get("cpd00048").get("name"));
}
@Test
void databaseTestSpeed() {
try {
LocalDateTime currentTime = LocalDateTime.now();
System.out.println(currentTime.getHour() + "h" + currentTime.getMinute() + "m" + currentTime.getSecond() + "s" );
GraphDatabaseService graphDatabaseService = TransytNeo4jInitializer.getDatabaseService(true, new Properties());
TransytGraphDatabaseService service = new TransytGraphDatabaseService(graphDatabaseService);
Transaction dataTx = graphDatabaseService.beginTx();
ResourceIterator<Node> allNodes = service.getAllNodes().iterator();
int i = 0;
while(allNodes.hasNext()) {
allNodes.next();
i++;
}
System.err.println(i);
dataTx.close();
service.shutdown();
graphDatabaseService.shutdown();
currentTime = LocalDateTime.now();
System.out.println(currentTime.getHour() + "h" + currentTime.getMinute() + "m" + currentTime.getSecond() + "s" );
}
catch (Exception e) {
e.printStackTrace();
}
}
// @Test
// void databaseTestSpeed() {
//
// try {
//
// LocalDateTime currentTime = LocalDateTime.now();
// System.out.println(currentTime.getHour() + "h" + currentTime.getMinute() + "m" + currentTime.getSecond() + "s" );
//
// GraphDatabaseService graphDatabaseService = TransytNeo4jInitializer.getDatabaseService(true, new Properties());
//
// TransytGraphDatabaseService service = new TransytGraphDatabaseService(graphDatabaseSe