From 80ef044a7c48f5a10c5056dbc46129230e2d3457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geovanny=20Gonz=C3=A1lez-Rodr=C3=ADguez?= Date: Mon, 20 Jan 2025 15:32:18 +0100 Subject: [PATCH] Format code with `black` Format some modules, in special the new ones. --- core/controller/ticket_controller.py | 1 + core/utils/das.py | 61 ++++++++++++++++------------ core/utils/dqm.py | 29 ++++++++----- 3 files changed, 56 insertions(+), 35 deletions(-) diff --git a/core/controller/ticket_controller.py b/core/controller/ticket_controller.py index edad349..c625ae3 100644 --- a/core/controller/ticket_controller.py +++ b/core/controller/ticket_controller.py @@ -1,6 +1,7 @@ """ Module that contains TicketController class """ + import json from copy import deepcopy from environment import ( diff --git a/core/utils/das.py b/core/utils/das.py index 2ce39e3..192f5a9 100755 --- a/core/utils/das.py +++ b/core/utils/das.py @@ -20,9 +20,10 @@ def get_lumi_ranges(i): result = [] for _, b in itertools.groupby(enumerate(i), lambda pair: pair[1] - pair[0]): b = list(b) - result.append([b[0][1],b[-1][1]]) + result.append([b[0][1], b[-1][1]]) return result + def das_do_command(query): """ A simple wrapper for dasgoclient. @@ -34,10 +35,13 @@ def das_do_command(query): list[str]: the dasgoclient command output split by newlines. """ - cmd = 'dasgoclient --query="%s"'%(query) - out = subprocess.check_output(cmd, shell=True, executable="/bin/bash").decode('utf8') + cmd = 'dasgoclient --query="%s"' % (query) + out = subprocess.check_output(cmd, shell=True, executable="/bin/bash").decode( + "utf8" + ) return out.split("\n") + def das_file_data(dataset): """ Given a dataset create a pandas DataFrame with the @@ -51,15 +55,16 @@ def das_file_data(dataset): - the file name; - the number of events in each file. """ - query = 'file dataset=%s | grep file.name, file.nevents'%(dataset) + query = "file dataset=%s | grep file.name, file.nevents" % (dataset) out = das_do_command(query) - out = [np.array(r.split(" "))[[0,3]] for r in out if len(r) > 0] + out = [np.array(r.split(" "))[[0, 3]] for r in out if len(r) > 0] - df = pd.DataFrame(out,columns=["file","events"]) + df = pd.DataFrame(out, columns=["file", "events"]) df.events = df.events.values.astype(int) return df + def das_lumi_data(dataset): """ Produces a file by file+lumi+run pandas DataFrame @@ -73,17 +78,17 @@ def das_lumi_data(dataset): - the lumisections. """ - query = 'file,lumi,run dataset=%s '%(dataset) + query = "file,lumi,run dataset=%s " % (dataset) out = das_do_command(query) - out = [r.split(" ") for r in out if len(r)>0] + out = [r.split(" ") for r in out if len(r) > 0] - df = pd.DataFrame(out,columns=["file","run","lumis"]) + df = pd.DataFrame(out, columns=["file", "run", "lumis"]) return df -def get_events_df(golden,dataset,events): +def get_events_df(golden, dataset, events): """ Produces a file by file pandas DataFrame @@ -104,9 +109,11 @@ def get_events_df(golden,dataset,events): lumi_df = das_lumi_data(dataset) file_df = das_file_data(dataset) - df = lumi_df.merge(file_df,on="file",how="inner") # merge file informations with run and lumis + df = lumi_df.merge( + file_df, on="file", how="inner" + ) # merge file informations with run and lumis df["lumis"] = [ - [int(ff) for ff in f.replace("[","").replace("]","").split(",")] + [int(ff) for ff in f.replace("[", "").replace("]", "").split(",")] for f in df.lumis.values ] @@ -123,29 +130,32 @@ def get_events_df(golden,dataset,events): if df_r["events"].sum() < 10000: continue - good_lumis = np.array([len([ll for ll in l if ll in golden[r]]) for l in df_r.lumis]) + good_lumis = np.array( + [len([ll for ll in l if ll in golden[r]]) for l in df_r.lumis] + ) n_lumis = np.array([len(l) for l in df_r.lumis]) - df_rs.append(df_r[good_lumis==n_lumis]) + df_rs.append(df_r[good_lumis == n_lumis]) - if len(df_rs)==0: + if len(df_rs) == 0: return pd.DataFrame([]) - if len(df_rs)==1: + if len(df_rs) == 1: df = df_rs else: df = pd.concat(df_rs) ## lumi sorting - df.loc[:,"min_lumi"] = [min(f) for f in df.lumis] - df.loc[:,"max_lumi"] = [max(f) for f in df.lumis] - df = df.sort_values(["run","min_lumi","max_lumi"]) + df.loc[:, "min_lumi"] = [min(f) for f in df.lumis] + df.loc[:, "max_lumi"] = [max(f) for f in df.lumis] + df = df.sort_values(["run", "min_lumi", "max_lumi"]) ## events skimming - df = df[df["events"] <= events] #jump too big files - df.loc[:,"sum_evs"] = df.loc[:,"events"].cumsum() + df = df[df["events"] <= events] # jump too big files + df.loc[:, "sum_evs"] = df.loc[:, "events"].cumsum() df = df[df["sum_evs"] < events] return df + def get_run_lumi(df): """ Produces the lumi mask dict starting from a pandas DataFrame @@ -170,17 +180,18 @@ def get_run_lumi(df): lumi_list = [ get_lumi_ranges( np.sort( - np.concatenate(df.loc[df["run"]==r,"lumis"].values).ravel() + np.concatenate(df.loc[df["run"] == r, "lumis"].values).ravel() ).tolist() ) for r in run_list ] - lumi_ranges = dict(zip(run_list,lumi_list)) + lumi_ranges = dict(zip(run_list, lumi_list)) return lumi_ranges -def get_lumi_dict(golden,dataset,events): + +def get_lumi_dict(golden, dataset, events): """ Produces a lumi mask for a given dataset, up to events, using a certification json @@ -197,7 +208,7 @@ def get_lumi_dict(golden,dataset,events): E.g. {run : [[lumi_1,lumi_2],[lumi_3,lumi_4]]} """ - df = get_events_df(golden,dataset,events) + df = get_events_df(golden, dataset, events) lumi = get_run_lumi(df) return lumi diff --git a/core/utils/dqm.py b/core/utils/dqm.py index f7cd030..f4da920 100644 --- a/core/utils/dqm.py +++ b/core/utils/dqm.py @@ -2,16 +2,17 @@ import json import os -#pylint: disable=import-error +# pylint: disable=import-error import numpy as np import requests from bs4 import BeautifulSoup from requests.exceptions import HTTPError -#pylint: enable=import-error +# pylint: enable=import-error base_cert_path = "/eos/user/c/cmsdqm/www/CAF/certification/" + def list_certification_files(cert_type): """ List all the certification files related to a certification type @@ -48,6 +49,7 @@ def list_certification_files(cert_type): return file_names + def get_certification_file(path): """ Get a certification file from the CMS DQM certification @@ -68,6 +70,7 @@ def get_certification_file(path): return file.json() + def get_cert_type(dataset): """ List all the certification files related to a certification type @@ -81,8 +84,8 @@ def get_cert_type(dataset): or Commisioning). """ - year = dataset.split("Run")[1][2:4] # from 20XX to XX - PD = dataset.split("/")[1] # pylint: disable=invalid-name + year = dataset.split("Run")[1][2:4] # from 20XX to XX + PD = dataset.split("/")[1] # pylint: disable=invalid-name cert_type = "Collisions" + str(year) if "Cosmics" in dataset: cert_type = "Cosmics" + str(year) @@ -93,7 +96,8 @@ def get_cert_type(dataset): return cert_type -def get_json_list(cert_type,web_fallback): + +def get_json_list(cert_type, web_fallback): """ List all the certification files related to a certification type either stored on CMS DQM EOS either, as a fallback, @@ -113,7 +117,9 @@ def get_json_list(cert_type,web_fallback): cert_path = base_cert_path + cert_type + "/" json_list = os.listdir(cert_path) json_list = [c for c in json_list if "Golden" in c and "era" not in c] - json_list = [c for c in json_list if c.startswith("Cert_C") and c.endswith("json")] + json_list = [ + c for c in json_list if c.startswith("Cert_C") and c.endswith("json") + ] ## ... if not we go to the website else: json_list = list_certification_files(cert_type=cert_type) @@ -127,6 +133,7 @@ def get_json_list(cert_type,web_fallback): return json_list + def get_golden_json(dataset): """ Output a the golden certification dictionary (json) for a specific datasets. @@ -148,12 +155,14 @@ def get_golden_json(dataset): cert_path = base_cert_path + cert_type + "/" web_fallback = not os.path.isdir(cert_path) - json_list = get_json_list(cert_type,web_fallback) + json_list = get_json_list(cert_type, web_fallback) # the larger the better, assuming file naming schema # Cert_X_RunStart_RunFinish_Type.json run_ranges = [int(c.split("_")[3]) - int(c.split("_")[2]) for c in json_list] - latest_json = np.array(json_list[np.argmax(run_ranges)]).reshape(1,-1)[0].astype(str) + latest_json = ( + np.array(json_list[np.argmax(run_ranges)]).reshape(1, -1)[0].astype(str) + ) best_json = str(latest_json[0]) if not web_fallback: with codecs.open(cert_path + "/" + best_json, encoding="utf-8") as js: @@ -164,9 +173,9 @@ def get_golden_json(dataset): # golden json with all the lumisections one by one for k in golden: - R = [] # pylint: disable=invalid-name + R = [] # pylint: disable=invalid-name for r in golden[k]: - R = R + list(range(r[0], r[1] + 1)) # pylint: disable=invalid-name + R = R + list(range(r[0], r[1] + 1)) # pylint: disable=invalid-name golden_flat[k] = R return golden_flat