Git Lab CI for docker build enabled! You can enable it using .gitlab-ci.yml in your project. Check file template at https://gitlab.bio.di.uminho.pt/snippets/5

Commit e3ad0c9c authored by Davide Lagoa's avatar Davide Lagoa
Browse files

new types of reactions possible and debug

parent 9d81defd
......@@ -134,7 +134,7 @@ public class Blast implements Observer{
ConcurrentLinkedQueue<AlignmentCapsule> results = null;
if(sequences.keySet().size()>0)
results = run_similaritySearch.runBlastSearch(true, properties.getBlastEvalueThreshold(), properties.getBitScore(), properties.getQueryCoverage());
results = run_similaritySearch.runBlastSearch(properties.getBlastpExecutableAlias(), true, properties.getBlastEvalueThreshold(), properties.getBitScore(), properties.getQueryCoverage());
return results;
}
......
......@@ -66,42 +66,12 @@ public class BlastAlignmentTransyt extends Observable implements ModelAlignments
private Map<String, Integer> kegg_taxonomy_scores;
private Integer referenceTaxonomyScore;
private boolean forceBlast;
private String blastExe;
final static Logger logger = LoggerFactory.getLogger(BlastAlignmentTransyt.class);
/**
* Default values for evalueThreshold(1E-6), bitScoreThreshold(50), queryCoverageThreshold(0.80) and alignmentMinScore(0.0);
*
* @param queryFasta
* @param subjectFasta
* @param querySequences
* @param treshold
* @param transportersSearch
* @param cancel
* @param alignmentContainerSet
* @param jc
*/
public BlastAlignmentTransyt(boolean forceBlast, String queryFasta, String subjectFasta, Map<String,AbstractSequence<?>> querySequences, double treshold, boolean transportersSearch, AtomicBoolean cancel, ConcurrentLinkedQueue<AlignmentCapsule> alignmentContainerSet, JAXBContext jc){
this.forceBlast = forceBlast;
this.setEvalueThreshold(1E-6);
this.setBitScoreThreshold(50);
this.setQueryCoverageThreshold(0.80);
this.setAlignmentMinScore(0);
this.queryFasta = queryFasta;
this.subjectFasta = subjectFasta;
this.threshold = treshold;
this.isTransportersSearch = transportersSearch;
this.querySequences = querySequences;
this.alignmentContainerSet = alignmentContainerSet;
this.cancel = cancel;
this.jc = jc;
}
/**
* Default value for alignmentMinScore(0.0);
*
......@@ -117,10 +87,11 @@ public class BlastAlignmentTransyt extends Observable implements ModelAlignments
* @param alignmentContainerSet
* @param jc
*/
public BlastAlignmentTransyt(boolean forceBlast, String queryFasta, String subjectFasta, Map<String,AbstractSequence<?>> querySequences, double treshold, double evalueThreshold,
public BlastAlignmentTransyt(boolean forceBlast, String blastExe, String queryFasta, String subjectFasta, Map<String,AbstractSequence<?>> querySequences, double treshold, double evalueThreshold,
double bitScoreThreshold, double queryCoverageThreshold, boolean transportersSearch, AtomicBoolean cancel, ConcurrentLinkedQueue<AlignmentCapsule> alignmentContainerSet, JAXBContext jc){
this.forceBlast = forceBlast;
this.blastExe = blastExe;
this.setEvalueThreshold(evalueThreshold);
this.setBitScoreThreshold(bitScoreThreshold);
this.setQueryCoverageThreshold(queryCoverageThreshold);
......@@ -153,19 +124,24 @@ public class BlastAlignmentTransyt extends Observable implements ModelAlignments
outputFile = new File(blastOutputFolderPath.concat(outputFileName));
outputFile.getParentFile().mkdirs();
// System.out.println(outputFile.getAbsolutePath());
if(forceBlast) {
System.out.println("blastp -query " + this.queryFasta + " -subject "
+ this.subjectFasta + " -out " + outputFile.getAbsolutePath() + " -outfmt 5");
Process p = Runtime.getRuntime().exec("blastp -query " + this.queryFasta + " -subject "
+ this.subjectFasta + " -out " + outputFile.getAbsolutePath() + " -outfmt 5");
String command = this.blastExe + " -evalue " + this.evalueThreshold + " -query " + this.queryFasta + " -subject "
+ this.subjectFasta + " -out " + outputFile.getAbsolutePath() + " -outfmt 5";
logger.debug(command);
Process p = Runtime.getRuntime().exec(command);
p.waitFor();
}
else
logger.info("Blast skipped, searching cached results...");
if(outputFile.exists()){
......@@ -176,7 +152,7 @@ public class BlastAlignmentTransyt extends Observable implements ModelAlignments
}
else{
logger.warn("pt.uminho.ceb.biosystems.transyt.scraper.blast output .xml file wasn't generated on {}", outputFile.getAbsolutePath());
logger.warn("blast output .xml file wasn't generated on {}", outputFile.getAbsolutePath());
}
}
catch (UnknownHostException e2) {
......
......@@ -94,7 +94,7 @@ public class RunSimilaritySearchTransyt extends Observable implements Observer {
* @return
* @throws Exception
*/
public ConcurrentLinkedQueue<AlignmentCapsule> runBlastSearch(boolean isTransportersSearch, Double eValueThreshold, Double bitScoreThreshold, Double queryCoverageThreshold) throws Exception {
public ConcurrentLinkedQueue<AlignmentCapsule> runBlastSearch(String blastExe, boolean isTransportersSearch, Double eValueThreshold, Double bitScoreThreshold, Double queryCoverageThreshold) throws Exception {
List<Thread> threads = new ArrayList<Thread>();
// ConcurrentLinkedQueue<String> queryArray = new ConcurrentLinkedQueue<String>(this.querySequences.keySet());
......@@ -130,15 +130,8 @@ public class RunSimilaritySearchTransyt extends Observable implements Observer {
ModelAlignments blastAlign;
if(eValueThreshold!=null && bitScoreThreshold!=null && queryCoverageThreshold!=null){
blastAlign = new BlastAlignmentTransyt(forceBlast, queryFilesPaths.get(i), this.subjectFastaFilePath, queriesSubSetList.get(i),
blastAlign = new BlastAlignmentTransyt(forceBlast, blastExe, queryFilesPaths.get(i), this.subjectFastaFilePath, queriesSubSetList.get(i),
this.similarity_threshold, eValueThreshold, bitScoreThreshold, queryCoverageThreshold, isTransportersSearch, this.cancel, alignmentContainerSet, jc);
}
else{
blastAlign= new BlastAlignmentTransyt(forceBlast, queryFilesPaths.get(i), this.subjectFastaFilePath, queriesSubSetList.get(i),
this.similarity_threshold, isTransportersSearch, this.cancel, alignmentContainerSet, jc);
}
if(eValueThreshold!=null)
((BlastAlignmentTransyt) blastAlign).setEvalueThreshold(eValueThreshold);
......
......@@ -8,22 +8,22 @@ import java.util.Set;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Relationship;
import org.neo4j.graphdb.RelationshipType;
import org.neo4j.graphdb.Transaction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import pt.uminho.ceb.biosystems.transyt.service.biosynth.initializeNeo4jdb;
import pt.uminho.ceb.biosystems.transyt.service.containers.BiosynthMetabolites;
import pt.uminho.ceb.biosystems.transyt.service.reactions.IdentifyReactionsMetabolites;
import pt.uminho.ceb.biosystems.transyt.service.reactions.TransportReactionsBuilder;
import pt.uminho.ceb.biosystems.transyt.service.transytDatabase.PopulateTransytNeo4jDatabase;
import pt.uminho.ceb.biosystems.transyt.utilities.capsules.BiosynthMetaboliteProperties;
import pt.uminho.ceb.biosystems.transyt.utilities.capsules.ReactionContainer;
import pt.uminho.ceb.biosystems.transyt.utilities.capsules.TcNumberContainer2;
import pt.uminho.ceb.biosystems.transyt.utilities.enumerators.MetaboliteReferenceDatabaseEnum;
import pt.uminho.ceb.biosystems.transyt.utilities.files.JSONFilesUtils;
import pt.uminho.ceb.biosystems.transyt.utilities.transytUtilities.Properties;
import pt.uminho.sysbio.biosynth.integration.io.dao.neo4j.MetaboliteMajorLabel;
import pt.uminho.sysbio.biosynth.integration.io.dao.neo4j.MetabolitePropertyLabel;
import pt.uminho.sysbio.biosynth.integration.neo4j.BiodbMetaboliteNode;
import pt.uminho.sysbio.biosynthframework.BiodbGraphDatabaseService;
......@@ -50,10 +50,10 @@ public class WriteByMetabolitesID {
Map<String, BiosynthMetaboliteProperties> data = getBiosynthDBData(service);
// Map<String, BiosynthMetaboliteProperties> data = null;
// @SuppressWarnings("resource")
// Scanner reader = new Scanner(System.in);
@SuppressWarnings("resource")
Scanner reader = new Scanner(System.in);
// System.out.println("Writing Excel...");
// System.out.println("Writing Excel...");
// WriteExcel.writeNeo4jKeggInfo(data);
logger.info("Retrieving data from Biosynth database by metabolite name...");
......@@ -75,27 +75,29 @@ public class WriteByMetabolitesID {
//
// Map<String, String[]> forChildsSearch = metabolitesIdentification.getforChildsSearch();
// int n = 1;
Map<String, Set<TcNumberContainer2>> newData = test(namesAndIDsContainer, data, service, reactionsData, properties); //uncomment
int n = 1;
// while (n != 99) {
//
// n = 99;
while (n != 99) {
test(namesAndIDsContainer, data, service, null, reactionsData, null, properties); //uncomment
//
//// test2(service);
n = 99;
// test2(service, data, namesAndIDsContainer);
new PopulateTransytNeo4jDatabase(data, newData, properties); //uncomment
//
// System.out.println("Enter a random number to repeat (100 to repeat data retrieval) or 99 to finish: ");
System.out.println("Enter a random number to repeat (100 to repeat data retrieval) or 99 to finish: ");
// try {
// n = reader.nextInt();
// } catch (Exception e) {
// e.printStackTrace();
//
// n = reader.nextInt();
// }
//
// }
try {
n = reader.nextInt();
} catch (Exception e) {
e.printStackTrace();
n = reader.nextInt();
}
}
dataTx.failure();
dataTx.close();
service.shutdown();
......@@ -108,16 +110,77 @@ public class WriteByMetabolitesID {
}
}
private static void test2(BiodbGraphDatabaseService service) {
private static void test2(BiodbGraphDatabaseService service, Map<String, BiosynthMetaboliteProperties> data,
BiosynthMetabolites namesAndIDsContainer) {
Node node = service.getNodeByEntryAndLabel("META:Modified-Sugars", MetaboliteMajorLabel.MetaCyc);
// int i = 0;
//
// for(String key : data.keySet()) {
//
// if(i == 100)
// break;
// else
// System.out.println(key);
//
// i++;
//
// }
if(node != null) {
// Node node = service.getMetaboliteProperty("photon", MetabolitePropertyLabel.Name);
try {
// Node node1 = service.getMetaboliteProperty("e", MetabolitePropertyLabel.Name);
//
//
// if(node1 != null)
// {
// System.out.println(node1.getAllProperties());
//
//
//
// Iterable<RelationshipType> rels = node1.getRelationships();
//
// for(RelationshipType rel : rels) {
// System.out.println(rel.name());
// }
// }
Node node = service.getNodeByEntryAndLabel("e", MetaboliteMajorLabel.BiGGMetabolite);
System.out.println(node.getAllProperties());
System.out.println(node.getProperty("name").toString().matches("(?i).*Modified.*"));
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// System.out.println(node.getAllProperties());
// System.out.println(node.getRelationships());
System.out.println("done!");
// try {
// for(Relationship r : node.getRelationships()) {
// System.out.println(r.getId());
// System.out.println(r.getAllProperties());
// System.out.println(r.getOtherNode(node).getAllProperties());
// }
// }
// catch (Exception e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
// Node node = service.getNodeByEntryAndLabel("META:Modified-Sugars", MetaboliteMajorLabel.MetaCyc);
//
// if(node != null) {
// System.out.println(node.getAllProperties());
// System.out.println(node.getProperty("name").toString().matches("(?i).*Modified.*"));
//
// }
// Iterable<RelationshipType> rels = node.getRelationshipTypes();
//
// for(RelationshipType rel : rels) {
......@@ -162,30 +225,24 @@ public class WriteByMetabolitesID {
public static void test(BiosynthMetabolites namesAndIDsContainer, Map<String, BiosynthMetaboliteProperties> data, BiodbGraphDatabaseService service, Map<String, Map<String, MetaboliteMajorLabel>> tcdbMetabolitesIDs,
Map<String, Set<TcNumberContainer2>> reactionsData, Map<String, String[]> forChildsSearch, Properties properties) {
public static Map<String, Set<TcNumberContainer2>> test(BiosynthMetabolites namesAndIDsContainer, Map<String, BiosynthMetaboliteProperties> data, BiodbGraphDatabaseService service,
Map<String, Set<TcNumberContainer2>> reactionsData, Properties properties) {
// new FetchCompoundsByName(service, false).getResults();
try {
// Map<String, Set<TcNumberContainer2>> reactionsData2 = new HashMap<>();
//
// String[] accessions = new String[] {"Q2QKM2"};
////
// String[] accessions = new String[] {"Q2RIS7"};
//
// for(String acc : accessions)
// reactionsData2.put(acc, reactionsData.get(acc));
// Map<String, Set<TcNumberContainer2>> reactionsData3 = new HashMap<>();
// reactionsData3.put("O51235", reactionsData.get("O51235"));
/////TRANSYT
Map<String, Set<TcNumberContainer2>> newData = new TransportReactionsBuilder(reactionsData, service, data, namesAndIDsContainer, properties).getResults(); //uncomment
// System.out.println(service.getNodeById(Long.valueOf(65729)).getAllProperties());
//
// for(TcNumberContainer2 container : newData.get("G8IIS8")) {
// for(TcNumberContainer2 container : newData.get("Q2RIS7")) {
//
// System.out.println(container.getTcNumber());
//
......@@ -212,9 +269,6 @@ public class WriteByMetabolitesID {
// test2(service);
new PopulateTransytNeo4jDatabase(data, newData, properties); //uncomment
// JSONFilesUtils.writeJSONTriageReactions(newData);
// Map<String, Set<TcNumberContainer2>> newData2 = new Has
......@@ -538,12 +592,14 @@ public class WriteByMetabolitesID {
//
// FindMetabolitesID.saveInfoInFile2("C:\\\\Users\\\\Davide\\\\Documents\\\\InternalDB\\\\AllBiosynthMetabolites.txt", allMetabolites);
System.out.println("Done!!!");
return newData;
}
catch (Exception e) {
e.printStackTrace();
}
return null;
}
......
package pt.uminho.ceb.biosystems.transyt.service.kbase;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import pt.uminho.ceb.biosystems.transyt.utilities.files.FilesUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import pt.uminho.ceb.biosystems.mew.biocomponents.container.Container;
import pt.uminho.ceb.biosystems.mew.biocomponents.container.components.CompartmentCI;
import pt.uminho.ceb.biosystems.mew.biocomponents.container.components.MetaboliteCI;
import pt.uminho.ceb.biosystems.mew.biocomponents.container.io.readers.JSBMLReader;
import pt.uminho.ceb.biosystems.transyt.utilities.files.ReadExcelFile;
public class Tools {
private static final Logger logger = LoggerFactory.getLogger(Tools.class);
/**
* @param path
......@@ -40,70 +41,40 @@ public class Tools {
*/
public static Set<String> readModelMetabolitesFromSBML(String path) {
if(path == null)
return null;
Set<String> allMetabolites = new HashSet<>();;
Map<String, String> metabolites = new HashMap<>();
try {
BufferedReader br = new BufferedReader(new FileReader(path));
String line = br.readLine();
while (line != null) {
if(line.contains("<species ")) {
JSBMLReader reader = new JSBMLReader(path, "NoName");
Container cont = new Container(reader);
Map<String, MetaboliteCI> metabolites = cont.getMetabolites();
Map<String, CompartmentCI> compartments = cont.getCompartments();
for(String metabolite : metabolites.keySet()) {
metabolite = metabolite.replaceAll("^M_", "");
String metaboliteAux = metabolite;
for(String compartment : compartments.keySet()) {
String id = null;
String formula = null;
Pattern p;
Matcher m;
p = Pattern.compile("id=\".*_(\\S*)_.*\"");
m = p.matcher(line);
if(m.find()) {
try {
id = m.group(1);
}
catch (Exception e) {
id = null;
}
}
p = Pattern.compile("chemicalFormula=\"(\\S*)\"");
m = p.matcher(line);
if(m.find()) {
try {
formula = m.group(1);
}
catch (Exception e) {
formula = null;
}
}
if(id != null && formula != null) {
metabolites.put(id, formula);
metabolite = metabolite.replaceAll("_".concat(compartment).concat("$"), "");
if(!metabolite.equals(metaboliteAux)) {
allMetabolites.add(metabolite);
break;
}
}
line = br.readLine();
}
br.close();
}
catch (Exception e) {
System.out.println("MODEL NOT PROVIDED!!!!!");
return null;
}
return metabolites.keySet();
catch (Exception e) {
logger.error("An error occurred while reading the model!!!! Searching metabolites.txt file...");
// e.printStackTrace();
}
return allMetabolites;
}
}
......@@ -38,12 +38,8 @@ public class IdentifyReactionsMetabolites {
this.dictionary = new Synonyms();
// Set<String> tcdbMetabolites = getMetabolitesFromReactions(reactionsData);
metabolites = getMetabolitesFromReactions(reactionsData);
// System.out.println(metabolites);
for(String metabolite : metabolites.keySet()) {
for(String originalMetabolite : metabolites.get(metabolite)) {
this.metabolitesByOriginalName.put(originalMetabolite, metabolite);
......@@ -359,6 +355,9 @@ public class IdentifyReactionsMetabolites {
private void identificationByDirectMatch() {
for(String metabolite : new HashSet<>(metabolites.keySet())) {
if(metabolite.equals("light"))
System.out.println();
try {
if(metabolite.equalsIgnoreCase("galactonate"))
......
......@@ -202,8 +202,6 @@ public class ProvideTransportReactionsToGenes {
resultsByEvalue = getReactionsForGenesByEvalue();
service.createContraints();
Set<String> tcNumbers = identifyTcNumbersForSearch();
// this.subunits.setSubunitsDatabase(findSubunitsInDatabase(tcNumbers));
......@@ -570,8 +568,8 @@ public class ProvideTransportReactionsToGenes {
Set<String> reactions = reactionsByTcNumberForAnnotation.get(tcNumber);
if(!reactions.isEmpty())
System.out.println(reactions);
// if(!reactions.isEmpty())
// System.out.println(reactions);
// System.out.println("dddd " + reactions);
......@@ -1654,7 +1652,7 @@ public class ProvideTransportReactionsToGenes {
// System.out.println(compounds);
// System.out.println("ToAdd " + reactionIDaux);
if((modelMetabolites == null || modelMetabolites.containsAll(compounds)) && reaction != null) { //null in case no sbml was inserted (accept all)
if((modelMetabolites == null || modelMetabolites.containsAll(compounds)) && reaction != null) { //null in case no sbml was inserted (accept all)
set.add(reactionID);
......
......@@ -98,8 +98,7 @@ public class TransportReactionsBuilder {
mappingModelSeed.put("META:CU+", "cpd30760"); //apagar depois de corrigir os mapeamentos
mappingModelSeed.put("META:CPD0-2124", "cpd04761"); //apagar depois de corrigir os mapeamentos
// System.out.println("estou aqui!");
mappingBiGG.put("cpd12713", "e"); //apagar depois de corrigir os mapeamentos
generateTransportReactions(reactionsData, biosynthMetabolites, service);
......@@ -137,12 +136,17 @@ public class TransportReactionsBuilder {
Synonyms dictionary = new Synonyms();
int reactionsCounter = 0;
Set<String> uniqueReactions = new HashSet<>();
Set<Integer> uniqueReactionsHashCode = new HashSet<>();
// int i = 0;
for(String accession : reactionsData.keySet()) {