DepUpdater/MainProcess.py at artifact · ruisearch/DepUpdater · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
"""Main process of the tool"""
import os
import sys
import csv
import argparse

from preprocess import exec_maven_command
from preprocess.Restore import Restore
from traverse.traverse import Traverse
from constants import set_log_path, set_soot_empty_cases_csv, RET_DIR, DATA_DIR
from evaluation.tech_lag import TechLag
from evaluation.dep_count import count_deps
from preprocess import multiModule
from logger.logger import log_debug


def expand_resolve_abspath(path):
    """expand the path to absolute path"""
    expanded_path = os.path.expanduser(path)
    resolved_path = os.path.normpath(expanded_path)
    absolute_path = os.path.abspath(resolved_path)
    return absolute_path

args_parser = argparse.ArgumentParser()
args_parser.add_argument('-r', '--root', help='the path to the cloned folder')
args_parser.add_argument('-m', '--module', help='the relative path to the module')
args_parser.add_argument('-j', '--jar', help='the relative path to the client jar')
args_parser.add_argument('-l','--local_dep_jar', nargs='+', help='the relative paths to the local module jar depended by client')
args = args_parser.parse_args()
## preprocess
## input : the path to the cloned folder
## output ： client jar/dependencies jar(from dependency tree)
path_to_folder = expand_resolve_abspath(args.root)
path_to_pom = os.path.join(path_to_folder, "pom.xml")
# only handle the module in relative_path_to_module;'.' means the pom of the module is just at the root directory of project
relative_path_to_module = args.module
relative_path_to_module = relative_path_to_module.removeprefix('./')

# get all local module
local_module_inform = {}
if args.local_dep_jar:
    MULTI_MODULE_FLAG = True
else:
    # no local dependencies, so no need to consider multi-module
    MULTI_MODULE_FLAG = False
if MULTI_MODULE_FLAG:
    module_paths = multiModule.get_all_module(path_to_folder)
    for module_path in module_paths:
        # local_module_inform is a mapping from local module's gav to its relative path
        local_module_inform.update(multiModule.get_gav(module_path, path_to_folder))

# set path to log file and soot empty cases file
repo_name = os.path.basename(path_to_folder)
if relative_path_to_module == '.':
    # if the module is located at the root directory of the project
    # store the result of it in _ folder
    data_dir = os.path.join(RET_DIR, repo_name, '_')
else:
    data_dir = os.path.join(RET_DIR, repo_name, relative_path_to_module)
log_path = os.path.join(data_dir, 'log.txt')
set_log_path(log_path)
soot_empty_csv = os.path.join(data_dir, 'soot_empty_cases.csv')
set_soot_empty_cases_csv(soot_empty_csv)
# remove existing soot empty cases file
if os.path.exists(soot_empty_csv):
    os.remove(soot_empty_csv)
# remove existing log file
if os.path.exists(log_path):
    os.remove(log_path)
# create the directory for result if not exist
if not os.path.exists(os.path.dirname(log_path)):
    os.makedirs(os.path.dirname(log_path))
# remove all the outdated files in the result directory if exist
for file in os.listdir(data_dir):
    os.remove(os.path.join(data_dir, file))

print("\n****** preprocessing ... ******\n")
log_debug(f"Start preprocessing for {repo_name}/{relative_path_to_module}")

# set the git repository to the last tag status
# status_command = f'cd {path_to_folder} && git add . && git reset --hard && git fetch --tags && git tag --sort=-creatordate | head -1 | xargs git checkout'
status_command = f'cd {path_to_folder} && git add . && git stash && git stash clear'
print("****** set the git repository to the lastest tag status ******")
os.system(status_command)
print("\n****** preprocess.package ... ******\n")
exit_code = exec_maven_command.mvn_package(path_to_folder, relative_path_to_module)
if exit_code != 0:
    print("\n****** module package fails. Please check the project ******\n")
    exit()
print("\n****** preprocess.package done! ******\n")
exit_code = exec_maven_command.mvn_test(path_to_folder, relative_path_to_module)
if exit_code != 0:
    print("\n****** module test fails. Please check the project ******\n")
    exit()

# execute mvn dependency:tree to generate dependency tree file in convenience of extracting GAV of dependencies
# result is in ./data/preprocess/dependency_tree.txt
print("\n****** get tree ... ******\n")
# exec_maven_command.mvn_dependency_tree(path_to_folder, relative_path_to_module)
# tree_file : path to the file containing resulting tree
tree_file, exit_code = exec_maven_command.mvn_verbose_dependency_tree(path_to_folder, relative_path_to_module)
if exit_code != 0:
    print("\n****** module dependency tree fails. Please check the project ******\n")
    exit()
print("\n****** tree got! ******\n")
# parse dependency_tree.txt to get GAV of client jar and dependencies jar,
# then download dependencies jar and copy client jar
print("\n****** restore dependency to graph ... ******\n")
graph = Restore(path_to_folder, relative_path_to_module, tree_file, local_module_inform, args.local_dep_jar)
json_path, original_json_path, original_tech_lag, local_dep_gav = graph.restore(args.jar)
print("\n****** dependency graph got! ******\n")
print("\n****** preprocess done! ******\n")
log_debug('\npreprocess done\n')
# count the number of dependencies before update
original_dep_count = count_deps(original_json_path)

# if the original tech lag is 0, then the module is already up-to-date
if original_tech_lag[0] == 0:
    print("\n ****** The module is already up-to-date ******\n")
    log_debug('The module is already up-to-date')
    print("\n****** result ******\n")
    print('compile success:', True)
    print('test pass:', True)
    print('original technical lag:', 0)
    print('current technical lag:', 0)
    print('reduced technical lag:', 0)
    print('original dependency count:', original_dep_count)
    print('current dependency count:', original_dep_count)
    print('reduced dependency count:', 0)
    exit()

# Traverse the dependency graph to compute the newest compatible version of each dependency
log_debug(f"Start traversing for {repo_name}/{relative_path_to_module}")
tra = Traverse(json_path, path_to_folder, relative_path_to_module)
compile_flag, test_flag = tra.traverse(local_dep_gav)

# compute the technical lag of the module
print("\n****** compute technical lag ... ******\n")
log_debug(f"Start computing tech lag for {repo_name}/{relative_path_to_module}")
lag = TechLag(json_path)
# record the original lag, current lag and reduced lag in a csv file
lag_csv_path = os.path.join(DATA_DIR, 'lag.csv')
with open(lag_csv_path, 'a') as f:
    writer = csv.writer(f)
    writer.writerow([repo_name, relative_path_to_module, original_tech_lag[0], lag.current_lag[0], \
        original_tech_lag[0] - lag.current_lag[0]])
            # original_tech_lag[1] - lag.current_lag[1],\
            #     original_tech_lag[2] - lag.current_lag[2],\
            #         original_tech_lag[3] - lag.current_lag[3],\
            #             original_tech_lag[4] - lag.current_lag[4],\
            #                 original_tech_lag[5] - lag.current_lag[5],\
            #                     original_tech_lag[6] - lag.current_lag[6],\
            #                         original_tech_lag[7] - lag.current_lag[7],\
            #                             original_tech_lag[8] - lag.current_lag[8],\
            #                                 original_tech_lag[9] - lag.current_lag[9],\
            #                                     original_tech_lag[10] - lag.current_lag[10],\
            #                                         original_tech_lag[11] - lag.current_lag[11]])

# count the number of dependencies after update
current_dep_count = count_deps(json_path)
# print the result of the tool
print("\n****** result ******\n")
print('compile success:', compile_flag)
print('test pass:', test_flag)
print('original technical lag:', original_tech_lag[0])
print('current technical lag:', lag.current_lag[0])
print('reduced technical lag:', original_tech_lag[0] - lag.current_lag[0])
print('original dependency count:', original_dep_count)
print('current dependency count:', current_dep_count)
print('reduced dependency count:', original_dep_count - current_dep_count)
# print('reduced technical lag in depth 1:', original_tech_lag[1] - lag.current_lag[1])
# print('reduced technical lag in depth 2:', original_tech_lag[2] - lag.current_lag[2])
# print('reduced technical lag in depth 3:', original_tech_lag[3] - lag.current_lag[3])
# print('reduced technical lag in depth 4:', original_tech_lag[4] - lag.current_lag[4])
# print('reduced technical lag in depth 5:', original_tech_lag[5] - lag.current_lag[5])
# print('reduced technical lag in depth 6:', original_tech_lag[6] - lag.current_lag[6])
# print('reduced technical lag in depth 7:', original_tech_lag[7] - lag.current_lag[7])
# print('reduced technical lag in depth 8:', original_tech_lag[8] - lag.current_lag[8])
# print('reduced technical lag in depth 9:', original_tech_lag[9] - lag.current_lag[9])
# print('reduced technical lag in depth 10:', original_tech_lag[10] - lag.current_lag[10])
# print('reduced technical lag in depth >10:', original_tech_lag[11] - lag.current_lag[11])