Skip to content

Issue #820 #846

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LakatosMark1 wants to merge 1 commit into
base: feature/csharp_plugin
Choose a base branch
from
Open

Issue #820 #846

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions plugins/csharp/parser/include/csharpparser/csharpparser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,10 @@ class CsharpParser : public AbstractParser

bool acceptProjectBuildPath(const std::string& buildDir_);
bool parseProjectBuildPath(
const std::vector<std::string>& path_,
const std::string& buildPath_);
void addSource(const std::string& filepath_, bool error_);
const std::vector<std::string>& path_ //,
); //const std::string& buildPath_
//void addSource(const std::string& filepath_, bool error_);
void addSource(const std::string& filepath_, const std::string& targetDll_, bool error_);
};

} // parser
Expand Down
70 changes: 40 additions & 30 deletions plugins/csharp/parser/src/csharpparser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,7 @@ CsharpParser::CsharpParser(ParserContext& ctx_): AbstractParser(ctx_)
{
_threadNum = _ctx.options["jobs"].as<int>();
}
/*
bool CsharpParser::acceptProjectBuildPath(const std::vector<std::string>& path_)
{
return path_.size() >= 2 && fs::is_directory(path_[0]) && fs::is_directory(path_[1]);
}*/

bool CsharpParser::acceptProjectBuildPath(const std::string& buildPath_)
{
return fs::is_directory(buildPath_);
Expand All @@ -42,26 +38,24 @@ bool CsharpParser::parse()
bool success = true;

std::vector<std::string> paths = _ctx.options["input"].as<std::vector<std::string>>();
std::string buildPath = _ctx.options["build-dir"].as<std::string>();

if (acceptProjectBuildPath(buildPath))
if (!paths.empty())
{
LOG(debug) << "C# parser parse path: " << paths[0];
LOG(debug) << "Parsed csharp project build path: " << buildPath;
success = success && parseProjectBuildPath(paths, buildPath);
LOG(debug) << "C# parser parse path: " << paths.size();
success = success && parseProjectBuildPath(paths);
}
else
{
LOG(error) << "Build path must be a directory!";
LOG(error) << "No input directories provided for C# parser!";
success = false;
}

return success;
}

bool CsharpParser::parseProjectBuildPath(
const std::vector<std::string>& paths_,
const std::string& buildPath_)
const std::vector<std::string>& paths_ //,
)
{
namespace ch = std::chrono;
std::future<std::string> log;
Expand All @@ -80,8 +74,6 @@ bool CsharpParser::parseProjectBuildPath(
command.append("'");
command.append(_ctx.options["database"].as<std::string>());
command.append("' '");
command.append(buildPath_);
command.append("' '");
command.append(csharp_path.string());
command.append("' ");
command.append(std::to_string(_ctx.options["jobs"].as<int>()));
Expand All @@ -108,22 +100,35 @@ bool CsharpParser::parseProjectBuildPath(

std::string line;
std::stringstream log_str(log.get());
//LOG(warning) << log_str.str();
int countFull = 0, countPart = 0;

while(std::getline(log_str, line, '\n'))
{
if (line[0] == '+' || line[0] == '-')
{
addSource(line.substr(1), line[0] == '-');
if (line[0] == '+')
{
countFull++;
std::string content = line.substr(1); // We cut off the +/- sign

// Find the line (|) that separates the file and the DLL
size_t separatorPos = content.find('|');

if (separatorPos != std::string::npos) {
// If it exists, we split the text along |
std::string filepath = content.substr(0, separatorPos);
std::string targetDll = content.substr(separatorPos + 1);

// We clean up the spaces from the beginning
filepath.erase(0, filepath.find_first_not_of(" \t"));

addSource(filepath, targetDll, line[0] == '-');
}
else
{
countPart++;
else {
// Fallback if for some reason the DLL name was not sent by C#
content.erase(0, content.find_first_not_of(" \t"));
addSource(content, "Unknown.dll", line[0] == '-');
}

if (line[0] == '+') { countFull++; }
else { countPart++; }
}
}

Expand All @@ -138,12 +143,12 @@ bool CsharpParser::parseProjectBuildPath(
return result == 0;
}

void CsharpParser::addSource(const std::string& filepath_, bool error_)
void CsharpParser::addSource(const std::string& filepath_, const std::string& targetDll_, bool error_)
{
util::OdbTransaction transaction(_ctx.db);

model::BuildActionPtr buildAction(new model::BuildAction);
buildAction->command = " ";
buildAction->command = "dotnet build " + targetDll_; //buildAction->command = " ";
buildAction->type = model::BuildAction::Compile;

model::BuildSource buildSource;
Expand All @@ -154,12 +159,21 @@ void CsharpParser::addSource(const std::string& filepath_, bool error_)
buildSource.file->type = "CS";
buildSource.action = buildAction;


model::BuildTarget buildTarget;
buildTarget.action = buildAction;

buildTarget.file = _ctx.srcMgr.getFile(targetDll_);
buildTarget.file->type = "CS_DLL";

_ctx.srcMgr.updateFile(*buildSource.file);
_ctx.srcMgr.updateFile(*buildTarget.file);
_ctx.srcMgr.persistFiles();

transaction([&, this] {
_ctx.db->persist(buildAction);
_ctx.db->persist(buildSource);
_ctx.db->persist(buildTarget); //new!!
});
}

Expand All @@ -176,10 +190,6 @@ extern "C"
{
boost::program_options::options_description description("C# Plugin");

description.add_options()
("build-dir,b", po::value<std::string>()->default_value("Build directory"),
"The build directory of the parsed project.");

return description;
}

Expand Down
125 changes: 68 additions & 57 deletions plugins/csharp/parser/src_csharp/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,13 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using CSharpParser.model;
using System.Xml.Linq;

namespace CSharpParser
{
class Program
{
//private readonly CsharpDbContext _context;
private static List<string> _rootDir;
private static string _buildDir = "";
private static string _buildDirBase = "";
private static string _connectionString = "";

Expand All @@ -28,11 +27,10 @@ static int Main(string[] args)
try
{
_connectionString = args[0].Replace("'", "");
_buildDir = args[1].Replace("'", "");
_buildDirBase = args[2].Replace("'", "");
threadNum = int.Parse(args[3]);
_buildDirBase = args[1].Replace("'", ""); //indexes
threadNum = int.Parse(args[2]);

for (int i = 4; i < args.Length; ++i)
for (int i = 3; i < args.Length; ++i)
{
_rootDir.Add(args[i].Replace("'", ""));
}
Expand All @@ -42,44 +40,6 @@ static int Main(string[] args)
WriteLine("Error in parsing command!");
return 1;
}
/*if (args.Length < 3)
{
WriteLine("Missing command-line arguments in CSharpParser!");
return 1;
}
else if (args.Length == 3)
{
_connectionString = args[0].Replace("'", "");
_rootDir = args[1].Replace("'", "");
_buildDir = args[2].Replace("'", "");
}
else if (args.Length == 4)
{
_connectionString = args[0].Replace("'", "");
_rootDir = args[1].Replace("'", "");
_buildDir = args[2].Replace("'", "");
bool success = int.TryParse(args[3], out threadNum);
if (!success){
WriteLine("Invalid threadnumber argument! Multithreaded parsing disabled!");
}
}
else if (args.Length == 5)
{
_connectionString = args[0].Replace("'", "");
_rootDir = args[1].Replace("'", "");
_buildDir = args[2].Replace("'", "");
_buildDirBase = args[3].Replace("'", "");
bool success = int.TryParse(args[4], out threadNum);
if (!success)
{
WriteLine("Invalid threadnumber argument! Multithreaded parsing disabled!");
}
}
else if (args.Length > 5)
{
WriteLine("Too many command-line arguments in CSharpParser!");
return 1;
}*/

//Converting the connectionstring into entiy framwork style connectionstring
string csharpConnectionString = transformConnectionString();
Expand All @@ -91,21 +51,67 @@ static int Main(string[] args)
CsharpDbContext _context = new CsharpDbContext(options);
_context.Database.Migrate();


List<string> allFiles = new List<string>();
// This dictionary will remember which file belongs to which DLL
Dictionary<string, string> fileToTargetDll = new Dictionary<string, string>();

foreach (var p in _rootDir)
{
Console.WriteLine(p);
allFiles.AddRange(GetSourceFilesFromDir(p, ".cs"));
// We find all .csproj files
var csprojFiles = Directory.GetFiles(p, "*.csproj", SearchOption.AllDirectories);
foreach (var csproj in csprojFiles)
{
string projectDir = Path.GetDirectoryName(csproj);
// Default DLL name based on project file name
string targetDll = Path.GetFileNameWithoutExtension(csproj) + ".dll";

// we try to read the real AssemblyName from the XML
try {
XDocument doc = XDocument.Load(csproj);
var assemblyNameNode = doc.Descendants("AssemblyName").FirstOrDefault();
if (assemblyNameNode != null && !string.IsNullOrWhiteSpace(assemblyNameNode.Value))
{
targetDll = assemblyNameNode.Value + ".dll";
}
} catch { /* If we cannot read the XML, the default name will remain.*/ }

// search for C# files belonging to the project (filtering out the garbage)
var csFiles = Directory.GetFiles(projectDir, "*.cs", SearchOption.AllDirectories)
.Where(f => !f.Contains("/obj/") && !f.Contains("\\obj\\") &&
!f.Contains("/bin/") && !f.Contains("\\bin\\"));

foreach (var cs in csFiles)
{
// if a file is not already in it (to avoid duplication)
if (!fileToTargetDll.ContainsKey(cs))
{
fileToTargetDll[cs] = targetDll;
allFiles.Add(cs);
}
}
}
}
allFiles = allFiles.Distinct().ToList();

foreach (var f in allFiles)
{
WriteLine(f);
}
IEnumerable<string> assemblies = GetSourceFilesFromDir(_buildDir, ".dll");
IEnumerable<string> assemblies_base = assemblies;
if (args.Length == 5)
assemblies_base = GetSourceFilesFromDir(_buildDirBase, ".dll");


IEnumerable<string> assemblies_base = GetSourceFilesFromDir(_buildDirBase, ".dll"); //loading basic dlls

List<string> assemblies = new List<string>();
foreach (var p in _rootDir)
{
// We search for all .dll files in all input directories
assemblies.AddRange(GetSourceFilesFromDir(p, ".dll"));
}
// Let's keep only one of each DLL based on the file name!
assemblies = assemblies.GroupBy(x => System.IO.Path.GetFileName(x))
.Select(g => g.First())
.ToList();

List<SyntaxTree> trees = new List<SyntaxTree>();
foreach (string file in allFiles)
Expand All @@ -129,14 +135,14 @@ static int Main(string[] args)
compilation = compilation.AddReferences(MetadataReference.CreateFromFile(file));
}

var runtask = ParalellRun(csharpConnectionString, threadNum, trees, compilation);
var runtask = ParalellRun(csharpConnectionString, threadNum, trees, compilation, fileToTargetDll);
int ret = runtask.Result;

return 0;
}

private static async Task<int> ParalellRun(string csharpConnectionString, int threadNum,
List<SyntaxTree> trees, CSharpCompilation compilation)
List<SyntaxTree> trees, CSharpCompilation compilation, Dictionary<string, string> fileToTargetDll)
{
var options = new DbContextOptionsBuilder<CsharpDbContext>()
.UseNpgsql(csharpConnectionString)
Expand All @@ -156,7 +162,7 @@ private static async Task<int> ParalellRun(string csharpConnectionString, int th
WriteLine(threadNum);
for (int i = 0; i < maxThread; i++)
{
ParsingTasks.Add(ParseTree(contextList[i],trees[i],compilation,i));
ParsingTasks.Add(ParseTree(contextList[i],trees[i],compilation,i,fileToTargetDll));
}

int nextTreeIndex = maxThread;
Expand All @@ -169,7 +175,7 @@ private static async Task<int> ParalellRun(string csharpConnectionString, int th
if (nextTreeIndex < trees.Count)
{
ParsingTasks.Add(ParseTree(contextList[nextContextIndex],
trees[nextTreeIndex],compilation,nextContextIndex));
trees[nextTreeIndex],compilation,nextContextIndex, fileToTargetDll));
++nextTreeIndex;
}
}
Expand All @@ -183,15 +189,20 @@ private static async Task<int> ParalellRun(string csharpConnectionString, int th
}

private static async Task<int> ParseTree(CsharpDbContext context,
SyntaxTree tree, CSharpCompilation compilation, int index)
SyntaxTree tree, CSharpCompilation compilation, int index,
Dictionary<string, string> fileToTargetDll)
{
var ParsingTask = Task.Run(() =>
{
WriteLine("ParallelRun " + tree.FilePath);
SemanticModel model = compilation.GetSemanticModel(tree);
var visitor = new AstVisitor(context, model, tree);
visitor.Visit(tree.GetCompilationUnitRoot());
WriteLine((visitor.FullyParsed ? "+" : "-") + tree.FilePath);
visitor.Visit(tree.GetCompilationUnitRoot());

// Find the DLL name and append a | to the filename.
string target = fileToTargetDll.ContainsKey(tree.FilePath) ? fileToTargetDll[tree.FilePath] : "Unknown.dll";
WriteLine((visitor.FullyParsed ? "+" : "-") + tree.FilePath + "|" + target);

return index;
});
return await ParsingTask;
Expand Down
Loading