diff --git a/bda-controller/src/main/java/gr/ntua/ece/cslab/selis/bda/controller/Entrypoint.java b/bda-controller/src/main/java/gr/ntua/ece/cslab/selis/bda/controller/Entrypoint.java index 25eb9103..e072f4ab 100644 --- a/bda-controller/src/main/java/gr/ntua/ece/cslab/selis/bda/controller/Entrypoint.java +++ b/bda-controller/src/main/java/gr/ntua/ece/cslab/selis/bda/controller/Entrypoint.java @@ -38,12 +38,20 @@ /** * Created by Giannis Giannakopoulos on 8/31/17. + * This class represents the actual BDA server and contains all the functional components + * that need to be instantiated when the server is launched. */ public class Entrypoint { private final static Logger LOGGER = Logger.getLogger(Entrypoint.class.getCanonicalName()); public static Configuration configuration; - + /** + * This is the main method of the BDA server which is used when starting and stopping it to + * initialize or shutdown connections with the IAM server, the BDA database and the Pub/Sub + * server and also to start/stop the CronJob scheduler. + * @param args the BDA configuration file + * @throws SystemConnectorException + */ public static void main(String[] args) throws SystemConnectorException { if (args.length < 1) { LOGGER.log(Level.WARNING, "Please provide a configuration file as a first argument"); diff --git a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/DimensionTable.java b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/DimensionTable.java index 2acb4abd..06f498da 100644 --- a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/DimensionTable.java +++ b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/DimensionTable.java @@ -24,6 +24,12 @@ /** * Created by Giannis Giannakopoulos on 10/11/17. + * This class represents a Dimension (Entity) Table. + * + * Entity Tables are modeled as resources identified by a name, a schema, and the data they + * are storing. + * + * @see DimensionTableSchema */ @XmlRootElement(name = "DimensionTable") @XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) @@ -32,34 +38,60 @@ public class DimensionTable { private DimensionTableSchema schema; private List data; + /** + * A simple placeholder constructor. + */ public DimensionTable(){ this.data = new LinkedList<>();} + /** + * @param name the table name + * @param schema the table schema as defined in {@link DimensionTableSchema} + * @param data the table data in the form of a list (must comply with the schema in {@param schema}) + */ public DimensionTable(String name, DimensionTableSchema schema, List data) { this.name = name; this.schema = schema; this.data = data; } + /** + * @return + */ public DimensionTableSchema getSchema() { return schema; } + /** + * @param schema + */ public void setSchema(DimensionTableSchema schema) { this.schema = schema; } + /** + * @return + */ public List getData() { return data; } + /** + * @param data + */ public void setData(List data) { this.data = data; } + /** + * @return + */ public String getName() { return name; } + /** + * @param name + */ public void setName(String name) { this.name = name; } diff --git a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/DimensionTableSchema.java b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/DimensionTableSchema.java index 1477f8a2..fa842abe 100644 --- a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/DimensionTableSchema.java +++ b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/DimensionTableSchema.java @@ -24,6 +24,12 @@ /** * Created by Giannis Giannakopoulos on 10/11/17. + * This class represents the schema of a Dimension (Entity) Table. + * + * Entity Tables are stored in a RDBMS integrated with the BDA, therefore need to have their schema well defined according + * to SQL requirements. The minimum information required for the initialization of such resources is the column names and types + * as well as primary keys. + * */ @XmlRootElement(name = "DimensionTableSchema") @XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) @@ -34,32 +40,55 @@ public class DimensionTableSchema { public DimensionTableSchema(){ this.columnNames = new LinkedList<>();}; + /** + * @param columnNames a list including the names of the table columns + * @param columnTypes a {@link KeyValue} list including the name and type of each table column + * @param primaryKey the table primary key + */ public DimensionTableSchema(List columnNames, List columnTypes, String primaryKey) { this.columnNames = columnNames; this.columnTypes = columnTypes; this.primaryKey = primaryKey; } + /** + * @return a list including the names of the table columns + */ public List getColumnNames() { return columnNames; } + /** + * @param columnNames a list including the names of the table columns + */ public void setColumnNames(List columnNames) { this.columnNames = columnNames; } + /** + * @return a {@link KeyValue} list including the name and type of each table column + */ public List getColumnTypes() { return columnTypes; } + /** + * @param columnTypes a {@link KeyValue} list including the name and type of each table column + */ public void setColumnTypes(List columnTypes) { this.columnTypes = columnTypes; } + /** + * @return the table primary key + */ public String getPrimaryKey() { return primaryKey; } + /** + * @param primaryKey the table primary key + */ public void setPrimaryKey(String primaryKey) { this.primaryKey = primaryKey; } diff --git a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/Job.java b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/Job.java index dea75d8e..1966d0fc 100644 --- a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/Job.java +++ b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/Job.java @@ -28,8 +28,15 @@ import java.util.logging.Level; import java.util.logging.Logger; +/** + * This class represents a Job. Jobs are instanciated 'recipes' which are connected to a specific trigger + * (either an incoming message or a periodic recurring trigger). This class contains all meta-data a job requires to be + * executed. + * + */ public class Job implements Serializable { private final static Logger LOGGER = Logger.getLogger(Job.class.getCanonicalName()); + private final static int DEFAULT_VECTOR_SIZE = 10; private transient int id; @@ -100,9 +107,22 @@ public class Job implements Serializable { public Job() { } + /** + * Default constructor + * + * @param name job name + * @param description a textual description of the job + * @param active a boolean active/inactive switch + * @param messageTypeId the messageType ID as defined during the configuration process + * @param recipeId the recipe ID as defined during the configuration process + * @param scheduleInfo the time pattern to schedule the job if it is cron-based + * @param resultStorage a string to identify where the result of the job will be saved (kpidb/pubsub/hdfs) + * @param dependJobId the id of the parent job if its result must be used as an input + */ public Job(String name, String description, boolean active, Integer messageTypeId, int recipeId, String resultStorage, String scheduleInfo, Integer dependJobId) { + this.name = name; this.description = description; this.active = active; @@ -184,6 +204,9 @@ public void setJobType() { public void setDependJobId(Integer dependJobId) { this.dependJobId = dependJobId; } + /** + * @return the contents of a Job object as a string + */ @Override public String toString() { return "Job{" + @@ -202,9 +225,14 @@ public String toString() { '}'; } + /** + * @param slug the SCN slug as defined in the configuration process + * @return a list containing Job objects corresponding to existing jobs + * @throws SQLException + * @throws SystemConnectorException + */ public static List getJobs(String slug) throws SQLException, SystemConnectorException { - - PostgresqlConnector connector = (PostgresqlConnector ) + PostgresqlConnector connector = (PostgresqlConnector ) SystemConnector.getInstance().getDTconnector(slug); Connection connection = connector.getConnection(); @@ -241,8 +269,13 @@ public static List getJobs(String slug) throws SQLException, SystemConnecto return jobs; } + /** + * @param slug the SCN slug as defined in the configuration process + * @return a list containing Job objects corresponding to existing active jobs + * @throws SQLException + * @throws SystemConnectorException + */ public static List getActiveJobs(String slug) throws SQLException, SystemConnectorException { - PostgresqlConnector connector = (PostgresqlConnector ) SystemConnector.getInstance().getDTconnector(slug); @@ -279,8 +312,14 @@ public static List getActiveJobs(String slug) throws SQLException, SystemCo return jobs; } + /** + * @param slug the SCN slug as defined in the configuration process + * @param id an existing job ID + * @return a Job object corresponding to the job with the given ID + * @throws SQLException + * @throws SystemConnectorException + */ public static Job getJobById(String slug, int id) throws SQLException, SystemConnectorException { - PostgresqlConnector connector = (PostgresqlConnector ) SystemConnector.getInstance().getDTconnector(slug); @@ -320,9 +359,15 @@ public static Job getJobById(String slug, int id) throws SQLException, SystemCon throw new SQLException("Job object not found."); } + /** + * @param slug the SCN slug as defined in the configuration process + * @param id the message ID as defined in the configuration process + * @return a Job object corresponding to a job triggered by the message type with the given ID + * @throws SQLException + * @throws SystemConnectorException + */ public static Job getJobByMessageId(String slug, int id) throws SQLException, SystemConnectorException { - - PostgresqlConnector connector = (PostgresqlConnector ) + PostgresqlConnector connector = (PostgresqlConnector ) SystemConnector.getInstance().getDTconnector(slug); Connection connection = connector.getConnection(); diff --git a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/KeyValue.java b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/KeyValue.java index 4a80f48f..b3a3feb2 100644 --- a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/KeyValue.java +++ b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/KeyValue.java @@ -48,7 +48,7 @@ public KeyValue(String key, String value) { /** * Getter for the key - * @return + * @return key */ public String getKey() { return key; @@ -64,7 +64,7 @@ public void setKey(String key) { /** * Getter for the value - * @return + * @return value */ public String getValue() { return value; diff --git a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/MessageType.java b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/MessageType.java index ac0b21cd..e39a37f5 100644 --- a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/MessageType.java +++ b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/MessageType.java @@ -37,6 +37,10 @@ @XmlRootElement(name = "MessageType") @XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) public class MessageType implements Serializable { + /** + * This class implements the different message types which the BDA can receive. All incoming messages must be of a + * defined type with specific structure and content to be accepted and processed. + */ private final static Logger LOGGER = Logger.getLogger(MessageType.class.getCanonicalName()); private final static int DEFAULT_VECTOR_SIZE = 10; @@ -52,6 +56,15 @@ public class MessageType implements Serializable { public MessageType() { } + /** + * Default constructor + * @param name message type name + * @param description message type description + * @param active a boolean switch indicating whether a message type is active or not + * @param format the actual message format, used for validation + * @param externalConnectorId the ID of the connector responsible for pushing this message type into the BDA + * @param externalDatasource the external datasource that sends this message + */ public MessageType(String name, String description, boolean active, String format, Integer externalConnectorId, String externalDatasource) { this.name = name; this.description = description; @@ -109,6 +122,9 @@ public List getMessageColumns() { return columns; } + /** + * @return String representation of MessageType objects + */ @Override public String toString() { return "MessageType{" + @@ -179,6 +195,12 @@ public String toString() { private final static String DELETE_MESSAGE_QUERY = "DELETE FROM metadata.message_type WHERE id = ?;"; + /** + * @param slug The SCN slug as defined in the configuration process + * @return List of all message types existing in the specified SCN + * @throws SQLException + * @throws SystemConnectorException + */ public static List getMessageTypes(String slug) throws SQLException, SystemConnectorException { PostgresqlConnector connector = (PostgresqlConnector ) SystemConnector.getInstance().getDTconnector(slug); @@ -213,6 +235,13 @@ public static List getMessageTypes(String slug) throws SQLException return messageTypes; } + /** + * @param slug The SCN slug as defined in the configuration process + * @param external + * @return List of all active message types existing in the specified SCN + * @throws SQLException + * @throws SystemConnectorException + */ public static List getActiveMessageTypes(String slug, boolean external) throws SQLException, SystemConnectorException { PostgresqlConnector connector = (PostgresqlConnector ) SystemConnector.getInstance().getDTconnector(slug); @@ -249,6 +278,11 @@ public static List getActiveMessageTypes(String slug, boolean exter return messageTypes; } + /** + * @param slug The SCN slug as defined in the configuration process + * @return List of all message types existing in the specified SCN + * @throws SystemConnectorException + */ public static List getActiveMessageTypeNames(String slug) throws SystemConnectorException { PostgresqlConnector connector = (PostgresqlConnector) SystemConnector.getInstance().getDTconnector(slug); Connection connection = connector.getConnection(); @@ -289,6 +323,13 @@ public static Boolean checkExternalMessageTypesExist(String slug) throws SQLExce throw new SQLException("Malformed query to find messages with external connectors."); } + /** + * @param slug The SCN slug as defined in the configuration process + * @param name message type name + * @return The message type existing in the specified SCN which matches the provided name + * @throws SQLException + * @throws SystemConnectorException + */ public static MessageType getMessageByName(String slug, String name) throws SQLException, SystemConnectorException { PostgresqlConnector connector = (PostgresqlConnector) SystemConnector.getInstance().getDTconnector(slug); Connection connection = connector.getConnection(); @@ -320,6 +361,13 @@ public static MessageType getMessageByName(String slug, String name) throws SQLE throw new SQLException("MessageType object not found."); } + /** + * @param slug The SCN slug as defined in the configuration process + * @param id The message type ID + * @return The message type existing in the specified SCN which matches the provided ID + * @throws SQLException + * @throws SystemConnectorException + */ public static MessageType getMessageById(String slug, int id) throws SQLException, SystemConnectorException { PostgresqlConnector connector = (PostgresqlConnector) SystemConnector.getInstance().getDTconnector(slug); Connection connection = connector.getConnection(); diff --git a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/Recipe.java b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/Recipe.java index 82b5e62e..6e2d90da 100644 --- a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/Recipe.java +++ b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/Recipe.java @@ -40,6 +40,10 @@ import java.util.logging.Level; import java.util.logging.Logger; +/** + * This class represents a recipe. Recipes are resources in the form of executable scripts or binaries which can be applied + * on data as operators. This class encapsulates all required recipe meta-data for this abstraction to be functional. + */ public class Recipe implements Serializable { private final static Logger LOGGER = Logger.getLogger(Recipe.class.getCanonicalName()); private final static int DEFAULT_VECTOR_SIZE = 10; @@ -103,6 +107,15 @@ public class Recipe implements Serializable { public Recipe() {} + /** + * Default constructor + * @param name recipe name + * @param description recipe description + * @param languageId the supported language ID (eg. java, python) as defined in the configuration process + * @param executablePath the path to the executable (in the HDFS by default) + * @param engineId the execution engine ID as defined in the configuration process (spark by default) + * @param args necessary arguments for script execution + */ public Recipe(String name, String description, int languageId, String executablePath, int engineId, RecipeArguments args) { this.name = name; this.description = description; @@ -172,6 +185,9 @@ public void setExists(boolean exists) { this.exists = exists; } + /** + * @return a string representation of objects of this class + */ @Override public String toString() { return "Recipe{" + @@ -186,6 +202,12 @@ public String toString() { '}'; } + /** + * @param slug the SCN slug as defined in the configuration process + * @return a list of all defined recipes + * @throws SQLException + * @throws SystemConnectorException + */ public static List getRecipes(String slug) throws SQLException, SystemConnectorException { PostgresqlConnector connector = (PostgresqlConnector ) @@ -224,6 +246,13 @@ public static List getRecipes(String slug) throws SQLException, SystemCo return recipes; } + /** + * @param slug the SCN slug as defined in the configuration process + * @param id the desired recipe ID + * @return the recipe which corresponds to the given ID + * @throws SQLException + * @throws SystemConnectorException + */ public static Recipe getRecipeById(String slug, int id) throws SQLException, SystemConnectorException { PostgresqlConnector connector = (PostgresqlConnector ) SystemConnector.getInstance().getDTconnector(slug); @@ -259,6 +288,12 @@ public static Recipe getRecipeById(String slug, int id) throws SQLException, Sys } + /** + * @param slug the SCN slug as defined in the configuration process + * @param name the desired recipe name + * @return the recipe which corresponds to the given name + * @throws SystemConnectorException + */ public static Recipe getRecipeByName(String slug, String name) throws SystemConnectorException { PostgresqlConnector connector = (PostgresqlConnector ) SystemConnector.getInstance().getDTconnector(slug); diff --git a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/RecipeArguments.java b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/RecipeArguments.java index 9d0c4ad4..733c1ded 100644 --- a/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/RecipeArguments.java +++ b/bda-datastore/src/main/java/gr/ntua/ece/cslab/selis/bda/datastore/beans/RecipeArguments.java @@ -23,6 +23,11 @@ import java.util.LinkedList; import java.util.List; +/** + * This class includes information about the arguments that recipes require to be executed correctly. + * These arguments could be dimension table related, message type related or other, unrelated arguments. + * + */ @XmlRootElement(name = "RecipeArguments") @XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) public class RecipeArguments implements Serializable { @@ -30,36 +35,60 @@ public class RecipeArguments implements Serializable { private List message_types; private List other_args; + /** + * Default constructor + */ public RecipeArguments() { this.dimension_tables = new LinkedList<>(); this.message_types = new LinkedList<>(); this.other_args = new LinkedList<>(); } + /** + * @return List of dimension tables related to the specific object + */ public List getDimension_tables() { return dimension_tables; } + /** + * @param dimension_tables dimension table names to be associated with the recipe + */ public void setDimension_tables(List dimension_tables) { this.dimension_tables = dimension_tables; } + /** + * @return List of message types related to the specific object + */ public List getMessage_types() { return message_types; } + /** + * @param message_types Message types to be associated with the recipe + */ public void setMessage_types(List message_types) { this.message_types = message_types; } + /** + * @return List of other arguments related to the specific object + */ public List getOther_args() { return other_args; } + /** + * @param other_args Arguments unrelated to dimension tables or message types to be associated with the recipe + */ public void setOther_args(List other_args) { this.other_args = other_args; } + /** + * @return String representation of RecipeArgument objects + */ @Override public String toString() { return "RecipeArguments{" + diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/Configuration.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/Configuration.java index 21f5a3b6..6bd28205 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/Configuration.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/Configuration.java @@ -46,6 +46,9 @@ public class Configuration { public final AuthClientBackend authClientBackend; public final KPIBackend kpiBackend; + /** + * Class to hold the BDA server configuration. + */ public class Server { private String address; private Integer port; @@ -61,6 +64,11 @@ public Integer getPort() { return port; } } + + /** + * Class to hold the storage engines configuration. Includes the BDA db, + * the EventLog FS, the Dimension tables FS and the HDFS options. + */ public class StorageBackend { // TODO: Should add username/password for every StorageBackend. // Modify Constructor accordingly. @@ -108,6 +116,11 @@ public String getBdaDatabaseURL() { public String getHDFSPassword() { return hdfsPassword; } } + + /** + * Class to hold the execution engines configuration. Includes Spark and Livy options, + * as well as the storage folder for executables. + */ public class ExecutionEngine { private String sparkMaster; private String sparkDeployMode; @@ -147,6 +160,10 @@ public ExecutionEngine(){} public String getLivyURL() { return livyURL; } } + + /** + * Class to hold the Pub/Sub server configuration. + */ public class PubSubServer { private String authHash, certificateLocation; @@ -158,6 +175,10 @@ public PubSubServer(){ public String getCertificateLocation() { return certificateLocation; } } + + /** + * Class to hold the Pub/Sub subscriber configuration. + */ public class PubSubSubscriber { private String url; @@ -166,6 +187,10 @@ public PubSubSubscriber(){ } public String getUrl() { return url; } } + + /** + * Class to hold the IAM server configuration. + */ public class AuthClientBackend { private Boolean authEnabled; private String authServerUrl, realm, clientId, clientSecret, bdaUsername, bdaPassword; @@ -187,6 +212,9 @@ public AuthClientBackend() { } public String getBdaPassword() { return bdaPassword; } } + /** + * Class to hold the KPI service configuration. + */ public class KPIBackend { private String dbUrl, dbUsername, dbPassword; @@ -198,6 +226,11 @@ public class KPIBackend { } + /** + * This method is used by all the modules to get the existing configuration object. + * @return the Configuration object that contains all configuration options + * @throws IllegalStateException + */ public static Configuration getInstance() throws IllegalStateException { if (configuration == null) { throw new IllegalStateException("Configuration not initialized."); @@ -206,6 +239,9 @@ public static Configuration getInstance() throws IllegalStateException { return configuration; } + /** + * Default constructor. + */ private Configuration() { this.server = new Server(); this.storageBackend = new StorageBackend(); diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/SystemConnector.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/SystemConnector.java index eb9991d4..247dc89b 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/SystemConnector.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/SystemConnector.java @@ -27,6 +27,10 @@ import java.util.logging.Logger; import java.lang.UnsupportedOperationException; +/** + * This class is used by all the BDA modules to connect to the underlying storage engines. + * It keeps open connections to every database that the BDA contains. + */ public class SystemConnector { private final static Logger LOGGER = Logger.getLogger(SystemConnector.class.getCanonicalName()); private static Configuration configuration; @@ -38,8 +42,11 @@ public class SystemConnector { private HashMap dtConnectors; private HashMap kpiConnectors; - /** The constructor creates new connections for the EventLog FS, the Dimension - * tables FS and the KPI db per LL as well as the BDA db. **/ + /** + * Default constructor. + * It creates new connections for the BDA db and the HDFS and initializes + * objects to store the connections of the SCN databases. + */ private SystemConnector() throws SystemConnectorException { this.elConnectors = new HashMap(); this.dtConnectors = new HashMap(); @@ -61,6 +68,12 @@ private SystemConnector() throws SystemConnectorException { ); } + /** + * This method is used by all the BDA modules to retrieve and use the + * existing database connections. + * @return the SystemConnector object containing all the database connections. + * @throws SystemConnectorException + */ public static SystemConnector getInstance() throws SystemConnectorException { if (systemConnector == null){ systemConnector = new SystemConnector(); @@ -69,6 +82,12 @@ public static SystemConnector getInstance() throws SystemConnectorException { return systemConnector; } + /** + * The init method is called when the BDA server is launched. + * It loads the configuration and opens connections to all the databases. + * @param args the configuration file location + * @throws SystemConnectorException + */ public static void init(String args) throws SystemConnectorException { // parse configuration configuration = Configuration.parseConfiguration(args); @@ -81,6 +100,9 @@ public static void init(String args) throws SystemConnectorException { } } + /** + * This method is used to create new connections for the EventLog FS, the Dimension + * tables FS and the KPI db per LL. **/ private void initSCNconnections() throws SystemConnectorException { List SCNs = new LinkedList<>(); try { @@ -120,6 +142,14 @@ private void initSCNconnections() throws SystemConnectorException { } } + /** + * This method is used upon SCN creation to create the required databases + * and open connections to them. + * @param scn the ScnDbInfo object that contains information such as the slug + * and scn database name + * @throws SystemConnectorException + * @throws UnsupportedOperationException + */ public void createScnDatabase(ScnDbInfo scn) throws SystemConnectorException, UnsupportedOperationException { @@ -185,6 +215,14 @@ public void createScnDatabase(ScnDbInfo scn) kpiConnectors.put(scnSlug, kpiConnector); } + /** + * This method is used upon SCN deletion to delete the scn databases + * and close the open connections to them. + * @param scn the ScnDbInfo object that contains information such as the slug + * and scn database name + * @throws UnsupportedOperationException + * @throws SystemConnectorException + */ public void destroyScnDatabase(ScnDbInfo scn) throws UnsupportedOperationException, SystemConnectorException { @@ -227,6 +265,10 @@ public void destroyScnDatabase(ScnDbInfo scn) kpiConnectors.remove(scnSlug); } + /** + * This method is used when the BDA server is shutdown to close all open database + * connections. + */ public void close(){ bdaConnector.close(); @@ -241,22 +283,45 @@ public void close(){ } } + /** + * Retrieve the existing connection to the BDA db. + * @return the Connector object that contains the open connection + */ public Connector getBDAconnector() { return bdaConnector; } + /** + * Retrieve the existing connection to the HDFS. + * @return the Connector object that contains the open connection + */ public Connector getHDFSConnector() { return hdfsConnector; } + /** + * Retrieve the existing connection to an SCN's Event Log. + * @param SCN the scn slug + * @return the Connector object that contains the open connection + */ public Connector getELconnector(String SCN) { return elConnectors.get(SCN); } + /** + * Retrieve the existing connection to an SCN's Dimension Tables. + * @param SCN the scn slug + * @return the Connector object that contains the open connection + */ public Connector getDTconnector(String SCN) { return dtConnectors.get(SCN); } + /** + * Retrieve the existing connection an SCN's KPI database. + * @param SCN the scn slug + * @return the Connector object that contains the open connection + */ public Connector getKPIconnector(String SCN) { return kpiConnectors.get(SCN); } diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/Connector.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/Connector.java index 114bde5a..54816f9d 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/Connector.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/Connector.java @@ -34,6 +34,11 @@ import java.util.LinkedList; import java.util.List; +/** + * This class represents a Connector which is used by a SCN to receive messages from the + * Pub/Sub. The Connector contains information about a running Connector instance (address and port), + * metadata required for the connection and if it connects to an external Pub/Sub. + */ @XmlRootElement(name = "Connector") @XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) public class Connector implements Serializable { @@ -70,6 +75,14 @@ public class Connector implements Serializable { public Connector() { } + /** + * Default constructor. + * @param name a name for the connector + * @param address the address of the running connector instance + * @param port the port of the running connector instance + * @param metadata information required for initiating a connection with the Pub/Sub + * @param external whether the connector instance is connected to a SELIS or an external Pub/Sub server + */ public Connector(String name, String address, Integer port, ConnectorMetadata metadata, boolean external) { this.name = name; this.address = address; diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ConnectorMetadata.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ConnectorMetadata.java index 158d1bac..74a6f709 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ConnectorMetadata.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ConnectorMetadata.java @@ -22,6 +22,11 @@ import java.io.Serializable; import java.util.List; +/** + * This class represents a ConnectorMetadata which contains information + * required for connecting with the Pub/Sub server like the username, password and + * the available message topics i.e. datasources to listen to. + */ @XmlRootElement(name = "ConnectorMetadata") @XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) public class ConnectorMetadata implements Serializable { @@ -31,6 +36,12 @@ public class ConnectorMetadata implements Serializable { public ConnectorMetadata() {} + /** + * Default constructor. + * @param username + * @param password + * @param datasources + */ public ConnectorMetadata(String username, String password, List datasources) { this.username = username; this.password = password; diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ExecutionEngine.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ExecutionEngine.java index 1add5675..a0048604 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ExecutionEngine.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ExecutionEngine.java @@ -34,6 +34,9 @@ import java.util.logging.Level; import java.util.logging.Logger; +/** + * This class represents an Execution Engine that the BDA can use to execute Recipes. + */ @XmlRootElement(name = "ExecutionEngine") @XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) public class ExecutionEngine implements Serializable { @@ -59,6 +62,13 @@ public class ExecutionEngine implements Serializable { // Empty constructor public ExecutionEngine() {} + /** + * Default constructor. + * @param name a name for the execution engine + * @param engine_path the engine url used to submit jobs + * @param local_engine if the execution is local or distributed + * @param args arguments for configuring the execution engine + */ public ExecutionEngine(String name, String engine_path, boolean local_engine, String args) { this.name = name; this.engine_path = engine_path; diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ExecutionLanguage.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ExecutionLanguage.java index 7762fdb7..00bcd4dd 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ExecutionLanguage.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ExecutionLanguage.java @@ -34,6 +34,9 @@ import java.util.logging.Level; import java.util.logging.Logger; +/** + * This class represents an Execution Language that a BDA Recipe can be written in. + */ @XmlRootElement(name = "ExecutionLanguage") @XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) public class ExecutionLanguage implements Serializable { @@ -55,6 +58,10 @@ public class ExecutionLanguage implements Serializable { // Empty constructor public ExecutionLanguage() {} + /** + * Default constructor + * @param name the language name + */ public ExecutionLanguage(String name) { this.name = name; } @@ -110,6 +117,13 @@ public void save() throws SystemConnectorException, SQLException { LOGGER.log(Level.INFO, "SUCCESS: Insert Into execution languages. ID: "+this.id); } + + /** + * This method is used to get the supported execution languages. + * @return a list of the existing Execution Language objects + * @throws SQLException + * @throws SystemConnectorException + */ public static List getLanguages() throws SQLException, SystemConnectorException { PostgresqlConnector connector = (PostgresqlConnector ) SystemConnector.getInstance().getBDAconnector(); Connection connection = connector.getConnection(); @@ -136,6 +150,13 @@ public static List getLanguages() throws SQLException, System throw new SQLException("Failed to retrieve ExecutionLanguage info."); } + /** + * This method is used to get the actual language name by its registered id. + * @param id the language id in the BDA database + * @return the execution language object + * @throws SQLException + * @throws SystemConnectorException + */ public static ExecutionLanguage getLanguageById(int id) throws SQLException, SystemConnectorException { PostgresqlConnector connector = (PostgresqlConnector ) SystemConnector.getInstance().getBDAconnector(); diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ScnDbInfo.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ScnDbInfo.java index 08c10aac..60a659cc 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ScnDbInfo.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/beans/ScnDbInfo.java @@ -30,6 +30,11 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +/** + * This class represents a SCN created inside the BDA. It contains information such as + * the SCN slug, the SCN name and a description, the database name and the connector + * instance that this SCN uses. + */ @XmlRootElement(name = "ScnDbInfo") @XmlAccessorType(XmlAccessType.PUBLIC_MEMBER) public class ScnDbInfo implements Serializable { @@ -70,6 +75,14 @@ public class ScnDbInfo implements Serializable { public ScnDbInfo() { } + /** + * Default constructor. + * @param slug the SCN slug + * @param name a name for the SCN + * @param description a description for the SCN + * @param dbname the SCN database name + * @param connectorId the connector id that this SCN uses to interact with the Pub/Sub + */ public ScnDbInfo(String slug, String name, String description, String dbname, Integer connectorId) { this.slug = slug; this.name = name; diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/Connector.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/Connector.java index 35ee92ab..77bfe09e 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/Connector.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/Connector.java @@ -16,6 +16,10 @@ package gr.ntua.ece.cslab.selis.bda.common.storage.connectors; +/** General methods that a storage engine connector should implement. **/ public interface Connector { + /** + * Close the connection. + */ void close(); } diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/ConnectorFactory.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/ConnectorFactory.java index cff2828f..e4edabe2 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/ConnectorFactory.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/ConnectorFactory.java @@ -23,6 +23,9 @@ import gr.ntua.ece.cslab.selis.bda.common.Configuration; import gr.ntua.ece.cslab.selis.bda.common.storage.SystemConnectorException; +/** + * Class that creates a connection to a storage engine. + */ public class ConnectorFactory { private static ConnectorFactory connFactory; @@ -40,7 +43,14 @@ public static ConnectorFactory getInstance(){ } /** Depending on the FS string format initialize a connector from a different class. - * Connectors are implemented for four different filesystems: local, HBase, HDFS, PostgreSQL. **/ + * Connectors are implemented for four different filesystems: local, HBase, HDFS, PostgreSQL. + * @param fs the storage engine connection url + * @param username the username used to connect + * @param password the password used to connect + * @param configuration the BDA configuration object + * @return the Connector object created + * @throws SystemConnectorException + */ public Connector generateConnector(String fs, String username, String password, Configuration configuration) throws SystemConnectorException { @@ -67,7 +77,17 @@ public Connector generateConnector(String fs, String username, String password, } /** Creates a new database and specified schemas. - * Returns the jdbcUrl of the new database. **/ + * @param fs the storage engine connection url + * @param username the username used to connect + * @param password the username used to connect + * @param configuration the BDA configuration object + * @param owner the username of the database owner + * @param dbname the database name + * @param schemas the database schemas + * @return the Url of the new database. + * @throws SystemConnectorException + * @throws UnsupportedOperationException + **/ public static String createNewDatabaseWithSchemas(String fs, String username, String password, Configuration configuration, String owner, String dbname, Vector schemas) throws SystemConnectorException, UnsupportedOperationException { @@ -110,7 +130,16 @@ public static String createNewDatabaseWithSchemas(String fs, String username, St return databaseUrl; } - /** Destroys a database. **/ + /** Destroys a database. + * @param fs the storage engine connection url + * @param username the username used to connect + * @param password the username used to connect + * @param configuration the BDA configuration object + * @param owner the username of the database owner + * @param dbname the database name + * @throws UnsupportedOperationException + * @throws SystemConnectorException + */ public static void dropDatabase(String fs, String username, String password, Configuration configuration, String owner, String dbname) throws UnsupportedOperationException, SystemConnectorException { @@ -139,6 +168,11 @@ public static void dropDatabase(String fs, String username, String password, Con return; } + /** + * Method to identify the filesystem type (local, HBase, HDFS, PostgreSQL) from the provided engine url. + * @param fs the storage engine connection url + * @return an identifier for the filesystem + */ public static int getConnectorType(String fs) { if (fs.contains("hdfs")) { return ConnectorFactory.CONNECTOR_HDFS_TYPE; diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/HBaseConnector.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/HBaseConnector.java index 276222ea..79f211d8 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/HBaseConnector.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/HBaseConnector.java @@ -161,6 +161,16 @@ public static String createNamespace(String fs, String username, String password return fs + dbname; } + /** + * Deletes the namespace for a specific SCN at the Event Log database. + * @param fs The URL of the SCN's EventLog database. + * @param username The username to connect to the EventLog database. + * @param password The password to connect to the EventLog database. + * @param configuration The Global BDA configuration. + * @param dbname The name of the new Namespace to delete. + * @throws IOException + * @throws ServiceException + */ public static void dropNamespace(String fs, String username, String password, Configuration configuration, String dbname) throws IOException, ServiceException { // Initialize HBase Configuration. @@ -201,6 +211,9 @@ public static void dropNamespace(String fs, String username, String password, Co LOGGER.log(Level.INFO, "HBase namespace deleted."); } + /** + * Close the connection to HBase. + */ public void close(){ try { connection.close(); @@ -209,6 +222,10 @@ public void close(){ } } + /** + * Get the actual connection from the Connector object. + * @return the Connection object + */ public Connection getConnection() { return connection; } diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/HDFSConnector.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/HDFSConnector.java index c455738f..6fe4e9f2 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/HDFSConnector.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/HDFSConnector.java @@ -24,6 +24,17 @@ public class HDFSConnector implements Connector { private org.apache.hadoop.fs.FileSystem fileSystem; private org.apache.hadoop.conf.Configuration hadoopConfiguration; + /** + * Creates a new `HDFSConnector` instance and checks its availability. + * + * TODO: Should use `usename`, `password` for connection. + * + * @param fs The URL of the HDFS. + * @param username The username to connect to HDFS. + * @param password The password to connect to HDFS. + * @param configuration The Global BDA configuration. + * @throws IOException + */ public HDFSConnector(String fs, String username, String password, Configuration configuration) throws IOException { try { diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/LocalFSConnector.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/LocalFSConnector.java index 4870159d..054e232d 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/LocalFSConnector.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/LocalFSConnector.java @@ -21,8 +21,14 @@ public class LocalFSConnector implements Connector { private String FS; - // This method creates the filesystem folder using the 'FS' parameter. - // If this folder exists, it should be initially empty (before the bootstraping). + /** + * Creates a new `LocalFSConnector` instance. This method creates the + * filesystem folder using the 'FS' parameter. If this folder exists, + * it should be initially empty (before the bootstraping). + * @param FS The local folder to create. + * @param username The username. + * @param password The password. + */ public LocalFSConnector(String FS, String username, String password){ this.FS = FS; diff --git a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/PostgresqlConnector.java b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/PostgresqlConnector.java index 7d1dfadd..e0528af1 100644 --- a/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/PostgresqlConnector.java +++ b/common/src/main/java/gr/ntua/ece/cslab/selis/bda/common/storage/connectors/PostgresqlConnector.java @@ -36,9 +36,15 @@ public class PostgresqlConnector implements Connector { private final static String DROP_OPEN_CONNECTIONS_QUERY = "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM "+ "pg_stat_activity WHERE pg_stat_activity.datname = '%s' AND pid <> pg_backend_pid();"; - - // The method creates a connection to the database provided in the 'jdbcURL' parameter. - // The database should be up and running. + + /** + * Creates a new `PostgresqlConnector` instance, checks PostgreSQL availability and initializes a connection. + * @param JdbcURL The URL of the BDA db and the SCN's Dimension table database. + * @param Username The username to connect to the database. + * @param Password The password to connect to the database. + * @throws SQLException + * @throws ClassNotFoundException + */ public PostgresqlConnector(String JdbcURL, String Username, String Password) throws SQLException, ClassNotFoundException { this.jdbcURL = JdbcURL; this.username = Username; @@ -70,6 +76,16 @@ public PostgresqlConnector(String JdbcURL, String Username, String Password) thr } } + /** + * Creates a new database for the given SCN at the Dimension tables database. + * @param jdbcUrl The URL of the BDA db and the SCN's Dimension table database. + * @param username The username to connect to the database. + * @param password The password to connect to the database + * @param owner The username of the database owner. + * @param dbname The name of the new database to create. + * @return the JDBC URL for the new database + * @throws SQLException + */ public static String createDatabase(String jdbcUrl, String username, String password, String owner, String dbname) throws SQLException { Connection localConnection = null; @@ -98,6 +114,16 @@ public static String createDatabase(String jdbcUrl, String username, String pass return jdbcUrl + dbname; } + /** + * Create a new schema in the given SCN database. This is used to store the + * SCN metadata in a separate schema. + * @param jdbcUrl The full URL to the SCN's Dimension table database. + * @param username The username to connect to the database. + * @param password The password to connect to the database + * @param owner The username of the database owner. + * @param schema The name of the schema to create. + * @throws SQLException + */ public static void createSchema(String jdbcUrl, String username, String password, String owner, String schema) throws SQLException { Connection localConnection = null; @@ -123,6 +149,15 @@ public static void createSchema(String jdbcUrl, String username, String password } + /** + * Deletes the Dimension tables database of a given SCN. + * @param jdbcUrl The URL of the SCN's Dimension table database. + * @param username The username to connect to the database. + * @param password The password to connect to the database + * @param owner The username of the database owner. + * @param dbname The name of the new database to create. + * @throws SQLException + */ public static void dropDatabase(String jdbcUrl, String username, String password, String owner, String dbname) throws SQLException { Connection localConnection = null; @@ -185,6 +220,9 @@ public String getUsername() { return username; } + /** + * Close the connection to the PostgreSQL database. + */ public void close(){ try { connection.close();