Git Lab CI for docker build enabled! You can enable it using .gitlab-ci.yml in your project. Check file template at https://gitlab.bio.di.uminho.pt/snippets/5

load bigg dump

parent 35d519c5
#FileLock
#Fri Feb 14 21:34:16 GMT 2020
hostName=172.20.67.168
id=170459fa0e877bd437557b681643aec27173b63c970
method=file
server=172.20.67.168\:57629
This diff is collapsed.
#FileLock
#Tue Feb 25 10:24:01 GMT 2020
id=1707bdffadd2571405cf8a53c120a5a5cf7cdf6d902
method=file
#FileLock
#Tue Feb 25 09:11:13 GMT 2020
hostName=172.20.67.191
id=1707b9d56c7bde5555c84091a20567d7fe75c230fa0
method=file
server=172.20.67.191\:49473
#FileLock
#Tue Feb 25 09:12:12 GMT 2020
hostName=172.20.67.191
id=1707b9e381cbaa23f7cbd98bd593dd0a4e9a1c90af2
method=file
server=172.20.67.191\:49493
......@@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>pt.uminho.ceb.biosystems.merlin</groupId>
<artifactId>merlin-bigg</artifactId>
<version>4.0.0-SNAPSHOT</version>
<version>4.0.05-SNAPSHOT</version>
<packaging>jar</packaging>
<name>merlin-bigg</name>
<url>http://maven.apache.org</url>
......@@ -21,16 +21,16 @@
<slf4j.version>1.8.0-beta4</slf4j.version>
</properties>
<repositories>
<repository>
<id>nexus</id>
<name>Nexus BioSystems Server</name>
<url>https://nexus.bio.di.uminho.pt/content/groups/public/</url>
</repository>
<repository>
<id>sing-repository</id>
<name>SING repository</name>
<url>http://maven.sing-group.org/repository/maven/</url>
</repository>
<repository>
<id>nexus</id>
<name>Nexus BioSystems Server</name>
<url>https://nexus.bio.di.uminho.pt/content/groups/public/</url>
</repository>
<repository>
<id>sing-repository</id>
<name>SING repository</name>
<url>http://maven.sing-group.org/repository/maven/</url>
</repository>
</repositories>
<build>
<plugins>
......@@ -40,6 +40,15 @@
<version>2.3.1</version>
<configuration>
<outputDirectory>${merlin.plugins.directory}</outputDirectory>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<mainClass>pt.uminho.ceb.biosystems.merlin.bigg.blast.BiggMain</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
<plugin>
......@@ -102,15 +111,20 @@
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>pt.uminho.ceb.biosystems.merlin</groupId>
<groupId>${pt.uminho.ceb.biosystems.merlin.groupId}</groupId>
<artifactId>merlin-gui</artifactId>
<version>4.0.2-RELEASE</version>
<version>4.0.05-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>${pt.uminho.ceb.biosystems.merlin.groupId}</groupId>
<artifactId>merlin-core</artifactId>
<version>4.0.3-SNAPSHOT</version>
<version>4.0.05-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>${pt.uminho.ceb.biosystems.merlin.groupId}</groupId>
<artifactId>merlin-hibernate</artifactId>
<version>4.0.05-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>pt.uminho.ceb.biosystems.mew</groupId>
......@@ -127,6 +141,11 @@
<artifactId>merlin-alignments</artifactId>
<version>4.0.05-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>${pt.uminho.ceb.biosystems.merlin.groupId}</groupId>
<artifactId>merlin-services</artifactId>
<version>4.0.05-SNAPSHOT</version>
</dependency>
<!-- </dependency> <groupId>org.sing_group</groupId> <artifactId>gc4s</artifactId>
<version>1.2.2</version> </dependency> -->
</dependencies>
......
......@@ -70,9 +70,11 @@ public class BiggMain {
items = getRandomModels(workFolderID);
break;
}
String workspaceName = listArgs.get(5);
logger.info("Starting ProvideBiggReactionsToGenes");
new ProvideBiggReactionsToGenes(workFolderID, filePath, properties, option, items,"workspaceName");
new ProvideBiggReactionsToGenes(workFolderID, filePath, properties, option, items, workspaceName);
File unzippedResults = new File(workFolderID+"Bigg_Files/Results/");
......
......@@ -26,7 +26,7 @@ public class Properties {
private static final Logger logger = LoggerFactory.getLogger(Properties.class);
public Properties() {
String propertiesFilePath = "/workdir/" + "configurations.txt";
String propertiesFilePath = BiggUtils.getRootDirectory() + "configs/configurations.txt"; //Docker - BiggUtils.getRootDirectory() + "configs/"
allProperties = FileUtils.readPropertiesFile(false,propertiesFilePath);
boolean go = assignVariables();
......
package pt.uminho.ceb.biosystems.merlin.bigg.blast;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
......@@ -11,18 +13,32 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.lang.StringUtils;
import pt.uminho.ceb.biosystems.merlin.bigg.data.BiggDataCenter;
import pt.uminho.ceb.biosystems.merlin.core.containers.model.CompartmentContainer;
import pt.uminho.ceb.biosystems.merlin.core.containers.model.ReactionContainer;
import pt.uminho.ceb.biosystems.merlin.core.datatypes.WorkspaceInitialData;
import pt.uminho.ceb.biosystems.merlin.gui.utilities.MerlinUtils;
import pt.uminho.ceb.biosystems.merlin.gui.utilities.NewWorkspaceRequirements;
import pt.uminho.ceb.biosystems.merlin.gui.utilities.TimeLeftProgress;
import pt.uminho.ceb.biosystems.merlin.services.DatabaseServices;
import pt.uminho.ceb.biosystems.merlin.services.WorkspaceInitialDataServices;
import pt.uminho.ceb.biosystems.merlin.services.model.ModelCompartmentServices;
import pt.uminho.ceb.biosystems.merlin.services.model.ModelGenesServices;
import pt.uminho.ceb.biosystems.merlin.services.model.ModelMetabolitesServices;
import pt.uminho.ceb.biosystems.merlin.services.model.ModelReactionsServices;
import pt.uminho.ceb.biosystems.merlin.services.model.loaders.ModelDatabaseLoadingServices;
public class ProvideBiggReactionsToGenes {
private Map<String,Set<String>> blastResults;
......@@ -46,7 +62,7 @@ public class ProvideBiggReactionsToGenes {
getReactionIds(properties, option);
gprsMap = createGPRs();
updateDatabase();
loadDatabase();
......@@ -107,10 +123,6 @@ public class ProvideBiggReactionsToGenes {
String reaction = parsedRule.getKey();
String rule = parsedRule.getValue();
// Validate
// if(reaction.equals("FIRST") || reaction.equals("ONLYORS") || reaction.equals("ONLYAND") || reaction.equals("ORDOESNTEXIST") || reaction.equals("ANDDOESNTEXIST") || reaction.equals("ABCDE"))
// System.out.println();
//If the rule contains OR, split the rule into a list, in order to validate each option
if(rule.contains(" or ")) {
......@@ -450,17 +462,23 @@ public class ProvideBiggReactionsToGenes {
return executeDistributions(set1, set2, cycle);
}
private void updateDatabase() {
private void loadDatabase() {
try {
//AtomicBoolean cancel = new AtomicBoolean(false);
AtomicBoolean cancel = new AtomicBoolean(false);
// DatabaseServices.readxmldb(workspaceName, workFolderID + "Bigg_Files/BiggDatabaseDump/", cancel, this);
ModelCompartmentServices.removeCompartmentsNotInUse(workspaceName);
//WorkspaceInitialData databaseInitialData = WorkspaceInitialDataServices.retrieveAllData(workspaceName);
System.out.println("-----READING XML DB------");
DatabaseServices.generateDatabase(workspaceName);
DatabaseServices.dropConnection(workspaceName);
DatabaseServices.readxmldb(workspaceName, workFolderID + "Bigg_Files/BiggDatabaseDump/", cancel, null);
//updateDatabase();
//for cada reaction, adicionar os genes, e meter a reaction in model
......@@ -471,6 +489,40 @@ public class ProvideBiggReactionsToGenes {
}
private void updateDatabase() {
try {
//Update reactions
for(Entry<String,String> entry : gprsMap.entrySet()) {
List<Integer> idReactions = new ArrayList<>(ModelReactionsServices.getReactionID(entry.getKey(), workspaceName));
List<ReactionContainer> map = ModelReactionsServices.getReactions(workspaceName, false);
for(Integer reaction : idReactions) {
ModelReactionsServices.updateModelReactionInModelByReactionId(workspaceName, reaction, true);
}
}
}catch(Exception ex) {
ex.printStackTrace();
}
}
}
......@@ -25,9 +25,10 @@ public class BiggDataLoader {
public static AtomicBoolean cancel = new AtomicBoolean(false);
private static HashMap<String,List<String>> metabolitesAliases = new HashMap<String,List<String>>();
private static String currentModel="";
public static void loadData(String model, String databaseName) throws Exception {
currentModel = model;
UpdateBiggFiles dataCenter = new UpdateBiggFiles(model);
WorkspaceInitialData databaseInitialData = null;
......@@ -345,128 +346,146 @@ public class BiggDataLoader {
// container.setGeneRule(reactionResult.getGeneReactionRule());
container.setUpperBound(Double.valueOf(reactionResult.getUpperBound()));
container.setLowerBound(Double.valueOf(reactionResult.getLowerBound()));
String equation = reactionResult.getReactionString();
if(equation != null && equation != "") {
List<MetaboliteContainer> reacStoich = new ArrayList<MetaboliteContainer>();
List<MetaboliteContainer> prodStoich = new ArrayList<MetaboliteContainer>();
if(equation != null && equation != "") { //there were cases where the equation contained only the symbol
String[] codes = {" &#x2192; "," &#8652; ", " &#x2190; "};
String reactants,products;
String[]equationSplitted;
List<String> reactantsSplitted = new ArrayList<String>();
List<String> productsSplitted = new ArrayList<String>();
if(equation.contains(codes[0]))
equationSplitted = equation.split(codes[0]);
else if(equation.contains(codes[1]))
equationSplitted = equation.split(codes[1]);
else
equationSplitted = equation.split(codes[2]);
if(equationSplitted.length>1) {
reactants = equationSplitted[0];
products = equationSplitted[1];
if(reactants.contains(" + "))
reactantsSplitted = Arrays.asList(reactants.split("\\+"));
else
reactantsSplitted.add(reactants);
if(products.contains(" + "))
productsSplitted = Arrays.asList(products.split("\\+"));
if(!equation.equals(" &#x2192; ") && !equation.equals(" &#8652; ") && !equation.equals(" &#x2190; ")) {
List<MetaboliteContainer> reacStoich = new ArrayList<MetaboliteContainer>();
List<MetaboliteContainer> prodStoich = new ArrayList<MetaboliteContainer>();
String[] codes = {" &#x2192; "," &#8652; ", " &#x2190; "};
String reactants,products;
String[]equationSplitted;
List<String> reactantsSplitted = new ArrayList<String>();
List<String> productsSplitted = new ArrayList<String>();
if(equation.contains(codes[0]))
equationSplitted = equation.split(codes[0]);
else if(equation.contains(codes[1]))
equationSplitted = equation.split(codes[1]);
else
productsSplitted.add(products);
equationSplitted = equation.split(codes[2]);
if(equationSplitted.length>1) {
reactants = equationSplitted[0];
products = equationSplitted[1];
//Trim items
if(reactants.contains(" + "))
reactantsSplitted = Arrays.asList(reactants.split("\\+"));
else
reactantsSplitted.add(reactants);
reactantsSplitted.replaceAll(String::trim);
if(products.contains(" + "))
productsSplitted = Arrays.asList(products.split("\\+"));
else
productsSplitted.add(products);
productsSplitted.replaceAll(String::trim);
}else { //if there are only reactants
reactants = equationSplitted[0];
reactantsSplitted.add(reactants.trim());
}
//Trim items
if(equation.toString().contains(codes[0]))
equation = equation.toString().replace(codes[0], " → ");
else if(equation.toString().contains(codes[1]))
equation = equation.toString().replace(codes[1], " ⇌ ");
else
equation = equation.toString().replace(codes[2], " ← ");
reactantsSplitted.replaceAll(String::trim);
container.setEquation(equation);
List<String> reactantsFinal = new ArrayList<String>();
List<String> productsFinal = new ArrayList<String>();
for(String reactant : reactantsSplitted) {
if(reactant.contains(" ")) {
String r = reactant.split(" ")[1];
reactantsFinal.add(r);
}else
reactantsFinal.add(reactant);
productsSplitted.replaceAll(String::trim);
}
for(String product : productsSplitted) {
if(product.contains(" ")) {
String r = product.split(" ")[1];
productsFinal.add(r);
}else
productsFinal.add(product);
}
}else { //if there are only reactants
reactants = equationSplitted[0];
reactantsSplitted.add(reactants.trim());
}
if(equation.toString().contains(codes[0]))
equation = equation.toString().replace(codes[0], " → ");
else if(equation.toString().contains(codes[1]))
equation = equation.toString().replace(codes[1], " ⇌ ");
else
equation = equation.toString().replace(codes[2], " ← ");
for(BiggMetabolites biggMetabolite : reaction.getMetabolites()) {
container.setEquation(equation);
List<String> reactantsFinal = new ArrayList<String>();
List<String> productsFinal = new ArrayList<String>();
for(String reactant : reactantsSplitted) {
if(reactant.contains(" ")) {
String r = reactant.split(" ")[1];
reactantsFinal.add(r);
}else
reactantsFinal.add(reactant);
}
for(String product : productsSplitted) {
if(product.contains(" ")) {
String r = product.split(" ")[1];
productsFinal.add(r);
}else
productsFinal.add(product);
}
String biggMet = biggMetabolite.getBiggId()+"_"+biggMetabolite.getCompartmentBiggId();
String[] stoich = new String[] {biggMetabolite.getStoichiometry().toString()};
for(BiggMetabolites biggMetabolite : reaction.getMetabolites()) {
for(String reac : reactantsFinal) {
if(biggMet.equals(reac)) {
MetaboliteContainer newMet = new MetaboliteContainer(biggMetabolite.getBiggId()+"_"+biggMetabolite.getCompartmentBiggId());
newMet.setStoichiometric_coefficient(biggMetabolite.getStoichiometry());
String biggMet = biggMetabolite.getBiggId()+"_"+biggMetabolite.getCompartmentBiggId();
String[] stoich = new String[] {biggMetabolite.getStoichiometry().toString()};
for(String reac : reactantsFinal) {
if(biggMet.equals(reac)) {
MetaboliteContainer newMet = new MetaboliteContainer(biggMetabolite.getBiggId()+"_"+biggMetabolite.getCompartmentBiggId());
newMet.setStoichiometric_coefficient(biggMetabolite.getStoichiometry());
if(biggMetabolite.getCompartment_name().equals("inner mitochondrial compartment"))
newMet.setCompartment_name("inner mitochondrial");
else if(biggMetabolite.getCompartment_name().equals("mitochondrial intermembrane"))
newMet.setCompartment_name("mitochondrial intermemb");
else if(biggMetabolite.getCompartment_name().equals("intermembrane space of mitochondria"))
newMet.setCompartment_name("inter space of mito");
else
newMet.setCompartment_name(biggMetabolite.getCompartment_name());
reacStoich.add(newMet);
if(biggMetabolite.getCompartment_name().equals("inner mitochondrial compartment"))
newMet.setCompartment_name("inner mitochondrial");
else if(biggMetabolite.getCompartment_name().equals("mitochondrial intermembrane"))
newMet.setCompartment_name("mitochondrial intermemb");
else if(biggMetabolite.getCompartment_name().equals("intermembrane space of mitochondria"))
newMet.setCompartment_name("inter space of mito");
else
newMet.setCompartment_name(biggMetabolite.getCompartment_name());
reacStoich.add(newMet);
}
}
}
for(String prod : productsFinal) {
if(biggMet.equals(prod)) {
MetaboliteContainer newMet = new MetaboliteContainer(biggMetabolite.getBiggId()+"_"+biggMetabolite.getCompartmentBiggId());
newMet.setStoichiometric_coefficient(biggMetabolite.getStoichiometry());
if(biggMetabolite.getCompartment_name().equals("inner mitochondrial compartment"))
newMet.setCompartment_name("inner mitochondrial");
else if(biggMetabolite.getCompartment_name().equals("mitochondrial intermembrane"))
newMet.setCompartment_name("mitochondrial intermemb");
else if(biggMetabolite.getCompartment_name().equals("intermembrane space of mitochondria"))
newMet.setCompartment_name("inter space of mito");
else
newMet.setCompartment_name(biggMetabolite.getCompartment_name());
prodStoich.add(newMet);
for(String prod : productsFinal) {
if(biggMet.equals(prod)) {
MetaboliteContainer newMet = new MetaboliteContainer(biggMetabolite.getBiggId()+"_"+biggMetabolite.getCompartmentBiggId());
newMet.setStoichiometric_coefficient(biggMetabolite.getStoichiometry());
if(biggMetabolite.getCompartment_name().equals("inner mitochondrial compartment"))
newMet.setCompartment_name("inner mitochondrial");
else if(biggMetabolite.getCompartment_name().equals("mitochondrial intermembrane"))
newMet.setCompartment_name("mitochondrial intermemb");
else if(biggMetabolite.getCompartment_name().equals("intermembrane space of mitochondria"))
newMet.setCompartment_name("inter space of mito");
else
newMet.setCompartment_name(biggMetabolite.getCompartment_name());
prodStoich.add(newMet);
}
}
}
}
container.setReactantsStoichiometry(reacStoich);
container.setProductsStoichiometry(prodStoich);
}else {
String[] codes = {" &#x2192; "," &#8652; ", " &#x2190; "};
if(equation.toString().contains(codes[0]))
equation = equation.toString().replace(codes[0], " → ");
else if(equation.toString().contains(codes[1]))
equation = equation.toString().replace(codes[1], " ⇌ ");
else
equation = equation.toString().replace(codes[2], " ← ");
container.setEquation(equation);
}
container.setReactantsStoichiometry(reacStoich);
container.setProductsStoichiometry(prodStoich);
}
......
......@@ -275,8 +275,6 @@ public class BiggLoadMetabolicData implements Runnable{
// if((enzymesList = this.resultEnzymes.poll()) != null) {
for(ProteinContainer container : containers) {
if(container.getExternalIdentifier().equals("2.3.1.163"))
System.out.println(container.getExternalIdentifier());
this.metabolicDataLoader.loadBiggProtein(container);
}
// this.progress.setTime((GregorianCalendar.getInstance().getTimeInMillis()-this.startTime), this.datum.incrementAndGet(), this.dataSize.get(), "Loading Enzymes");
......
......@@ -25,6 +25,7 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import pt.uminho.ceb.biosystems.merlin.gui.utilities.NewWorkspaceRequirements;
import pt.uminho.ceb.biosystems.merlin.services.DatabaseServices;
import pt.uminho.ceb.biosystems.merlin.utilities.io.FileUtils;
......@@ -32,15 +33,16 @@ public class UpdateBiggFiles {
private static BiggModels model = new BiggModels();
private static HashMap<String,String> compartments = setCompartments();
private static boolean resultsNeedUpdate = false;
private static List<String> modelBiggIds = new ArrayList<String>();
//Docker Paths
private static String mainPath = BiggUtils.getRootDirectory(); //Main path where all the files will be
private static String mainPath = BiggUtils.getRootDirectory();//BiggUtils.getRootDirectory(); //Main path where all the files will be
private static String modelsPath = mainPath.concat("BiggData/"); //Folder where the models files will be
private static String resultsPath = mainPath.concat("Results/"); //Folder where the results filed will be
private static String resultsZipPath = mainPath.concat("Results.tgz"); //Results zip path
private static File modelsFile = new File(modelsPath);
private static File resultsFile = new File(resultsPath);
private static List<String> modelBiggIds = new ArrayList<String>();
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
......@@ -50,22 +52,25 @@ public class UpdateBiggFiles {
System.out.println("----starting get models details-----");
getModelsDetails(resultModel);
verifyResultsFolder();
System.out.println("----genes reaction relation-----");
genesReactionsRelation(resultModel);
System.out.println("----creating fasta file-----");
createFastaFile(resultModel);
System.out.println("----Seq Id reaction relation-----");
writeSeqIDReactionRelationFile();
System.out.println("----Genes reactions rule file-----");
genesReactionsRuleFile();
System.out.println("----Models organisms file-----");
getModelsOrganismsFile(resultModel);
zipResultsFolder();
exportBiggDatabase();
//if(resultsNeedUpdate) {
verifyResultsFolder();
System.out.println("----genes reaction relation-----");
genesReactionsRelation(resultModel);
System.out.println("----creating fasta file-----");
createFastaFile(resultModel);
System.out.println("----Seq Id reaction relation-----");
writeSeqIDReactionRelationFile();
System.out.println("----Genes reactions rule file-----");
genesReactionsRuleFile();
System.out.println("----Models organisms file-----");
getModelsOrganismsFile(resultModel);
zipResultsFolder();
exportBiggDatabase();
//}
......@@ -102,11 +107,6 @@ public class UpdateBiggFiles {
forceGetInfo = true;
}
//ALTERAR MODELOS PARA TESTAR
// modelBiggIds.clear();
// modelBiggIds.add("e_coli_core");
// modelBiggIds.add("iAB_RBC_283");
for (String model : modelBiggIds) {
//Get current files and check if the file already exists
......@@ -118,6 +118,8 @@ public class UpdateBiggFiles {
}
resultsNeedUpdate=true; //starts as false. If there is at least one model that needs update, change the flag to true, in order to indicate that the results need to be downloaded again
JSONObject fullDetailsModel = new JSONObject();
JSONObject fullDetailsModelComplete = new JSONObject();
......@@ -1028,12 +1030,13 @@ public class UpdateBiggFiles {
try {
String database = "biggData";
String database = "biggDatabaseDump";
DatabaseServices.generateDatabase(database);
searchAndLoad(database);
NewWorkspaceRequirements.injectRequiredDataToNewWorkspace(database);
searchAndLoad(database);
String path = mainPath + "BiggDatabaseDump/";
......
<plugin start="true">
<uid>merlin-bigg</uid>
<name>pt.uminho.ceb.biosystems.merlin.bigg</name>
<version>4.0.1</version>
<version>4.0.0</version>
<dependencies>
<dependency uid="merlin" />
......
<plugin start="true">
<uid>merlin-bigg</uid>
<name>pt.uminho.ceb.biosystems.merlin.bigg</name>
<version>4.0.1</version>
<version>4.0.0</version>
<dependencies>
<dependency uid="merlin" />
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment