Git Lab CI for docker build enabled! You can enable it using .gitlab-ci.yml in your project. Check file template at https://gitlab.bio.di.uminho.pt/snippets/5

Commit 5bb841d1 authored by Davide Lagoa's avatar Davide Lagoa
Browse files

logger and TRIAGE neo4j database

parent d0a77108
......@@ -15,14 +15,16 @@
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-10">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
......
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.codegen.targetPlatform=10
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.compliance=10
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.8
org.eclipse.jdt.core.compiler.release=enabled
org.eclipse.jdt.core.compiler.source=10
......@@ -9,42 +9,66 @@
</description>
<repositories>
<repository>
<repository>
<id>nexus</id>
<name>Nexus BioSystems Server</name>
<!-- <url>http://192.168.1.99/nexus/content/groups/public/</url> -->
<url>http://193.137.11.210/nexus/content/groups/public/</url>
</repository>
</repositories>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
</repository>
</repositories>
<build>
<plugins>
<!-- <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version> <configuration> <source>9</source> <target>9</target>
</configuration> </plugin> -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<release>10</release>
</configuration>
<dependencies>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
<version>6.2</version> <!-- Use newer version of ASM -->
</dependency>
</dependencies>
</plugin>
</plugins>
</build>
<dependencies>
<!-- <dependency>
<dependency>
<groupId>utilities</groupId>
<artifactId>triage-utilities</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency> -->
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>tcdb</groupId>
<artifactId>triage</artifactId>
<version>0.0.1-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>uk.ac.ebi.uniprot</groupId>
<artifactId>japi</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
......@@ -52,40 +76,67 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>pt.uminho.sysbio.biosynthframework</groupId>
<artifactId>biosynth-biodb</artifactId>
<artifactId>biosynth-biodb-chebi</artifactId>
<version>0.9.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<groupId>
pt.uminho.sysbio.biosynthframework
</groupId>
<artifactId>biosynth-biodb</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>pt.uminho.sysbio.biosynthframework</groupId>
<artifactId>biosynth-biodb-chebi</artifactId>
<version>0.9.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>pt.uminho.sysbio.biosynthframework</groupId>
<artifactId>biosynth-biodb-reactome</artifactId>
<version>0.9.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>pt.uminho.sysbio.biosynthframework</groupId>
<artifactId>biosynth-chemanalysis</artifactId>
<version>0.9.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>pt.uminho.sysbio.biosynthframework</groupId>
<artifactId>biosynth-core</artifactId>
<version>0.9.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......@@ -101,6 +152,10 @@
<groupId>org.biojava</groupId>
<artifactId>biojava-core</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
</exclusions>
</dependency>
......@@ -113,19 +168,35 @@
<groupId>org.neo4j</groupId>
<artifactId>neo4j</artifactId>
</exclusion>
<exclusion>
<groupId>
pt.uminho.sysbio.biosynthframework
</groupId>
<artifactId>biosynth-biodb</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j</artifactId>
<version>3.4.1</version>
<groupId>org.neo4j</groupId>
<artifactId>neo4j</artifactId>
<version>3.4.1</version>
</dependency>
<dependency>
<groupId>pt.uminho.sysbio.biosynthframework</groupId>
<artifactId>biosynth-visualization</artifactId>
<version>0.9.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......@@ -149,6 +220,12 @@
<groupId>org.biojava</groupId>
<artifactId>biojava-alignment</artifactId>
<version>5.0.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......@@ -156,13 +233,23 @@
<artifactId>json-simple</artifactId>
<version>1.1.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.8.0-beta1</version>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.1.7</version>
</dependency>
</dependencies>
</project>
......@@ -5,13 +5,18 @@ import java.util.Set;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Transaction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import edu.uminho.biosynth.core.data.integration.neo4j.HelperNeo4jConfigInitializer;
import internalDB.WriteByMetabolitesID;
import pt.uminho.sysbio.biosynth.integration.io.dao.neo4j.MetaboliteMajorLabel;
import pt.uminho.sysbio.biosynth.integration.neo4j.BiodbMetaboliteNode;
import pt.uminho.sysbio.biosynthframework.BiodbGraphDatabaseService;
public class initializeNeo4jdb {
private static final Logger logger = LoggerFactory.getLogger(initializeNeo4jdb.class);
public static void main(String[] args) {
......@@ -49,12 +54,12 @@ public class initializeNeo4jdb {
public static GraphDatabaseService getDataDatabase(GraphDatabaseService dataDb) {
if (dataDb == null) {
// logger.info("[INIT] graph database...");
System.out.println("[INIT] graph database...");
logger.info("[INIT] graph database...");
dataDb = HelperNeo4jConfigInitializer.initializeNeo4jDataDatabaseConstraints("C:\\Users\\Davide\\Documents\\BASE DE DADOS BIOSYNTH\\db3_0");
// logger.info("[INIT] graph database... done!");
System.out.println("[INIT] graph database... done!");
logger.info("[INIT] graph database... done!");
}
return dataDb;
}
......
package internalDB;
import java.io.File;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Observable;
import java.util.Observer;
import java.util.Scanner;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.biojava.nbio.core.sequence.io.FastaReaderHelper;
import org.biojava.nbio.core.sequence.template.AbstractSequence;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Relationship;
import org.neo4j.graphdb.RelationshipType;
import org.neo4j.graphdb.Transaction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import biosynth.initializeNeo4jdb;
import blast.Blast;
import containers.BiosynthMetabolites;
import enumerators.MetaboliteReferenceDatabaseEnum;
import files.FilesUtils;
import files.JSONFilesUtils;
import pt.uminho.ceb.biosystems.merlin.bioapis.externalAPI.ncbi.CreateGenomeFile;
//import pt.uminho.ceb.biosystems.merlin.local.alignments.core.RunSimilaritySearch;
import pt.uminho.ceb.biosystems.merlin.utilities.Enumerators.AlignmentScoreType;
import pt.uminho.ceb.biosystems.merlin.utilities.Enumerators.Method;
import pt.uminho.ceb.biosystems.merlin.utilities.containers.capsules.AlignmentCapsule;
import pt.uminho.sysbio.biosynth.integration.io.dao.neo4j.MetaboliteMajorLabel;
import pt.uminho.sysbio.biosynth.integration.neo4j.BiodbMetaboliteNode;
import pt.uminho.sysbio.biosynthframework.BiodbGraphDatabaseService;
......@@ -49,11 +36,12 @@ import utilities.triage_utilities.Utilities;
*/
public class WriteByMetabolitesID {
// private static final Logger logger = LoggerFactory.getLogger(WriteByMetabolitesID.class);
private static final Logger logger = LoggerFactory.getLogger(WriteByMetabolitesID.class);
public static void main(String[] args) {
try {
GraphDatabaseService graphDatabaseService = initializeNeo4jdb.getDataDatabase(null);
Transaction dataTx = graphDatabaseService.beginTx();
......@@ -61,10 +49,7 @@ public class WriteByMetabolitesID {
// BiodbGraphDatabaseService service = null;
// logger.info("Retrieving data...");
System.out.println("Retrieving data...");
// logger.trace("salfnafsn");
logger.info("Retrieving data from Biosynth database...");
// Map<String, BiosynthMetaboliteProperties> data = getBiosynthDBData(service);
Map<String, BiosynthMetaboliteProperties> data = null;
......@@ -74,7 +59,8 @@ public class WriteByMetabolitesID {
// System.out.println("Writing Excel...");
// WriteExcel.writeNeo4jKeggInfo(data);
System.out.println("Retrieving names...");
logger.info("Retrieving data from Biosynth database by metabolite name...");
BiosynthMetabolites namesAndIDsContainer = getBiosynthDataByName(service, true); //154225 names
// BiosynthMetabolites namesAndIDsContainer = null;
......@@ -138,18 +124,18 @@ public class WriteByMetabolitesID {
// Map<String, Set<TcNumberContainer2>> newData2 = new HashMap<>();
//
// newData2.put("Q8SKU2", newData.get("Q8SKU2"));
// newData2.put("Q9MUK5", newData.get("Q9MUK5"));
// newData2.put("Q9SC41", newData.get("Q9SC41"));
// newData2.put("O43715", newData.get("O43715"));
// newData2.put("O24303", newData.get("O24303"));
// newData2.put("Q41010", newData.get("Q41010"));
// newData2.put("Q41009", newData.get("Q41009"));
// newData2.put("P0C891", newData.get("P0C891"));
// newData2.put("Q02028", newData.get("Q02028"));
// newData2.put("O49931", newData.get("O49931"));
// newData2.put("Q9ZST8", newData.get("Q9ZST8"));
// newData2.put("Q9ZST9", newData.get("Q9ZST9"));
// newData2.put("P29897", newData.get("P29897"));
// newData2.put("C9RT46", newData.get("C9RT46"));
// newData2.put("A8IJF8", newData.get("A8IJF8"));
// newData2.put("A9VJD5", newData.get("A9VJD5"));
// newData2.put("P54854", newData.get("P54854"));
// newData2.put("Q93HR7", newData.get("Q93HR7"));
// newData2.put("O67092", newData.get("O67092"));
// newData2.put("Q9UL62", newData.get("Q9UL62"));
// newData2.put("Q9LQZ0", newData.get("Q9LQZ0"));
// newData2.put("O67095", newData.get("O67095"));
// newData2.put("P54862", newData.get("P54862"));
// newData2.put("P27243", newData.get("P27243"));
new PopulateTriageNeo4jDatabase(newData);
......
package reactions;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.Scanner;
import java.util.Set;
import connection.ReadTcdbFastaFile;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Transaction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import blast.Blast;
import enumerators.TypeOfTransporter;
import files.ReadExcelFile;
import internalDB.WriteByMetabolitesID;
import javafx.util.Pair;
import pt.uminho.ceb.biosystems.merlin.bioapis.externalAPI.ebi.uniprot.UniProtAPI;
import pt.uminho.ceb.biosystems.merlin.utilities.containers.capsules.AlignmentCapsule;
import tcdb.capsules.FastaTcdb;
import tcdb.capsules.GeneContainer;
import tcdb.capsules.TcdbGene;
import tcdb.capsules.ReactionContainer;
import triageDatabase.TriageGeneralProperties;
import triageDatabase.TriageGraphDatabaseService;
import triageDatabase.TriageNeo4jInitializer;
import triageDatabase.TriageNodeLabel;
import triageDatabase.TriageRelationshipType;
import uk.ac.ebi.kraken.interfaces.uniprot.NcbiTaxon;
import uk.ac.ebi.kraken.interfaces.uniprot.Organism;
import uk.ac.ebi.kraken.interfaces.uniprot.UniProtEntry;
public class ProvideTransportReactionsToGenes { //change this name, the ractions are already built
public class ProvideTransportReactionsToGenes {
private Map<String, List<AlignmentCapsule>> blastResults;
private TriageGraphDatabaseService service;
private Map<String, Set<String>> homologousGenes;
private String[] taxonomy;
private String organism;
private Map<String, Set<ReactionContainer>> reactionContainers;
private static final Logger logger = LoggerFactory.getLogger(WriteByMetabolitesID.class);
public ProvideTransportReactionsToGenes() {
homologousGenes = new HashMap<>();
taxonomy = "Eukaryota, Fungi, Dikarya, Ascomycota, Saccharomycotina, Saccharomycetes, Saccharomycetales, Saccharomycetaceae, Kluyveromyces".split(","); //usar taxonomy do renato
public static void main(String[] args) {
organism = "Kluyveromyces lactis (strain ATCC 8585 / CBS 2359 / DSM 70799 / NBRC 1267 / NRRL Y-1140 / WM37)";
try {
logger.info("TRIAGE initialized!");
@SuppressWarnings("resource")
Scanner reader = new Scanner(System.in);
Map<String, FastaTcdb> fastaTcdb = ReadTcdbFastaFile.readfasta(false); // align genome sequences against fastaTcdb sequences //change boolean //reader perde uma seqeuencia, ver o que se passa
int n = 1;
Blast blast = new Blast();
blastResults = blast.getAlignmentsByQuery();
if(fastaTcdb == null)
System.out.println("ERROR WHILE READING TCDB FASTA");
while (n != 99) {
List<String[]> table = ReadExcelFile.getData("C:\\Users\\Davide\\Documents\\reactionsBuilderTriage\\test.xlsx");
start();
Map<String, List<String>> homologousGenes = new HashMap<>();
// Map<String, String> testCounts = new TreeMap<>();
///////test
List<String> homologous = new ArrayList<>();
try {
n = reader.nextInt();
} catch (Exception e) {
e.printStackTrace();
n = reader.nextInt();
}
for(String[] line : table) {
homologous.add(line[1]);
// testCounts.put(line[1], line[4]);
}
// System.out.println(testCounts);
}
homologousGenes.put("KLLA0B00264g", homologous);
logger.info("TRIAGE shutdown!");
}
catch (Exception e) {
e.printStackTrace();
}
}
private void start() {
buildReactionContainersForBlastResults();
// List<String[]> table = ReadExcelFile.getData("C:\\Users\\Davide\\Documents\\reactionsBuilderTriage\\test.xlsx");
ProvideTransportReactionsToGenes.processHomologousGenesInfo(null, null, homologousGenes, fastaTcdb);
Map<String, GeneContainer> data = processHomologousGenesInfo();
}
/**
* Method to get all reaction Containers for all blast results.
*
* @return
*/
private void buildReactionContainersForBlastResults(){
GraphDatabaseService graphDatabaseService = TriageNeo4jInitializer.getDatabaseService();
service = new TriageGraphDatabaseService(graphDatabaseService);
logger.info("Beginning transaction with neo4j triage database...");
Transaction dataTx = graphDatabaseService.beginTx();
for(String queryAccession : blastResults.keySet()) {
Set<String> homologous = new HashSet<>();
for(AlignmentCapsule capsule : blastResults.get(queryAccession)) {
homologous.add(capsule.getTarget());
String tcNumber = capsule.getTcdbID();
Node tcNumberNode = service.findNode(TriageNodeLabel.TC_Number, TriageGeneralProperties.TC_Number.toString(), tcNumber);
if(tcNumberNode != null) {
Set<Node> reactionNodes = service.getAllRelatedNodes(tcNumberNode, TriageRelationshipType.has_reaction);
reactionContainers.put(queryAccession, getAllNodesAsReactionContainers(tcNumber, reactionNodes));
}
else
logger.warn("No results found in TRIAGE neo4j database for tcNumber {}! Please consider update the database.", tcNumber);
}
homologousGenes.put(queryAccession, homologous);
}
dataTx.close();
service.shutdown();
graphDatabaseService.shutdown();
logger.info("TRIAGE neo4j database shutdown...");
}
/**
......@@ -50,20 +151,14 @@ public class ProvideTransportReactionsToGenes { //change this name, the ractio
*
* @param homologousGenes
*/
private static void processHomologousGenesInfo(String[] taxonomy, String organism