update the output dir

Signed-off-by: bvolovat <[email protected]>

collect-metrics-job.yaml

@@ -19,81 +19,40 @@ data:
     echo "Starting setup_and_run.sh script..."
     START_TIME=$(date +%s)
 
-    # Add early logging of duration
-    echo "Duration time set to: ${DURATION_TIME} minutes"
-
-    echo "Updating package list and installing required packages..."
-    apt-get update
-    apt-get install -y git curl apt-transport-https ca-certificates
-
-    echo "Downloading and installing kubectl..."
-    curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
-    chmod +x kubectl
-    mv kubectl /usr/local/bin/
-
-    # Get Helm Chart Version
-    HELM_VERSION=$(kubectl get deployment -n kubescape -o jsonpath='{.items[0].metadata.labels.helm\.sh/chart}')
-
-    # Extract Image Versions (Excluding `http-request`)
-    IMAGE_VERSIONS=$(kubectl get pods -n kubescape -o jsonpath='{range .items[*]}{.metadata.name}{" -> "}{.spec.containers[*].image}{"\n"}{end}' |
-    awk -F'/' '{print $NF}' | awk -F':' '{if ($2 ~ /^v/) print $1": "$2; else print $1": v"$2}' |
-    sort -u)
-
-    # Extract only one `node-agent` version
-    NODE_AGENT_VERSION=$(echo "$IMAGE_VERSIONS" | grep "node-agent" | head -n 1)
-
-    WEBHOOK_URL="${WEBHOOK_URL}"
-
-    echo "Installing Python dependencies..."
-    pip install --no-cache-dir requests pandas matplotlib plotly numpy
-
-    echo "Decoding GIT_TOKEN..."
-    DECODED_TOKEN=$(echo -n "${GIT_TOKEN}" | tr -d '\n' | base64 -d | tr -d '\n')
-
-    echo "Cloning repository..."
+    echo "Cloning performance repository for scripts..."
     git clone "https://${DECODED_TOKEN}@github.com/armosec/perfornamce.git" /workspace
     cd /workspace
-
+    
+    echo "Cloning Logs repository (shallow clone)..."
+    git clone --depth 1 "https://${DECODED_TOKEN}@github.com/armosec/Logs.git" /workspace/Logs
+    
     echo "Configuring Git..."
     git config --global user.email "[email protected]"
     git config --global user.name "bvolovat"
-
-    echo "Waiting for ${EXACT_DURATION} minutes before running tests..."
-    sleep $((EXACT_DURATION * 60))
-
-    END_TIME=$(date +%s)
-    EXACT_DURATION=$(( ($END_TIME - $START_TIME) / 60 ))
-    export EXACT_DURATION
-
-    echo "Creating output directory..."
-    mkdir -p output
-
-    echo "Running tests with exact duration of ${EXACT_DURATION} minutes..."
+    
+    # Get Helm version
+    HELM_VERSION=$(kubectl get deployment -n kubescape -o jsonpath='{.items[0].metadata.labels.helm\.sh/chart}')
+    CLEAN_HELM_VERSION=$(echo "$HELM_VERSION" | sed 's/[^a-zA-Z0-9._-]/_/g')
+    
+    # Set up the output directory path with timestamp and helm version
+    TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
+    export OUTPUT_DIR="/workspace/Logs/performance-inCluster/${TIMESTAMP}_helm_${CLEAN_HELM_VERSION}"
+    echo "Setting OUTPUT_DIR environment variable: ${OUTPUT_DIR}"
+    echo "Creating output directory: ${OUTPUT_DIR}"
+    mkdir -p "${OUTPUT_DIR}"
+    
+    echo "Running tests with specified duration of ${DURATION_TIME} minutes..."
     python get_data_from_prometheus.py
-    python get_pprof.py 
-
-    echo "Pushing results to repository..."
-    git add output/
-    git commit -m "Add test results from cluster run - Duration: ${EXACT_DURATION}m"
+    python get_pprof.py
+    
+    # Change to Logs repo directory to commit and push
+    cd /workspace/Logs
+    
+    echo "Pushing results to Logs repository..."
+    git add "performance-inCluster/${TIMESTAMP}_helm_${CLEAN_HELM_VERSION}"
+    git commit -m "Add performance test results - Duration: ${DURATION_TIME}m, Helm: ${HELM_VERSION}"
     git push origin main
 
-    echo "Test execution complete."
-
-    curl -X POST --data-urlencode "payload={
-        \"channel\": \"C06AJ92GHLM\",
-        \"username\": \"Performance-test\",
-        \"icon_emoji\": \":military_helmet:\",
-        \"text\": \"*Performance Test Completed!*\nHelm Chart: \`$HELM_VERSION\`\nDuration: *${EXACT_DURATION} minutes*\n<https://github.com/armosec/perfornamce/tree/main/output|View outputs>\",
-        \"attachments\": [{
-            \"color\": \"#007bff\",
-            \"fields\": [{
-                \"title\": \"Image Versions\",
-                \"value\": \"\`\`\`$IMAGE_VERSIONS\`\`\`\",
-                \"short\": false
-            }]
-        }]
-    }" "$WEBHOOK_URL"
-
 ---
 
 apiVersion: batch/v1
@@ -121,7 +80,7 @@ spec:
             secretKeyRef:
               name: perf-channel-webhook
               key: url
-        - name: EXACT_DURATION
+        - name: DURATION_TIME
           value: "$(DURATION_TIME)"
         volumeMounts:
         - name: run-script

get_data_from_prometheus.py

@@ -25,13 +25,24 @@ class PrometheusConfig:
 class PrometheusMetricsCollector:
     def __init__(self, config: Optional[PrometheusConfig] = None):
         self.config = config or PrometheusConfig()
-        self.output_dir = "output"
-        os.makedirs(self.output_dir, exist_ok=True)
 
-        # Get exact duration from environment variable
+        # Get output directory from environment variable with 'output' as default
+        self.output_dir = os.getenv('OUTPUT_DIR', 'output')
+        logger.info(f"Using output directory: {self.output_dir}")
+
+        # Ensure the output directory exists
+        try:
+            os.makedirs(self.output_dir, exist_ok=True)
+            logger.info(f"Successfully created/verified output directory: {self.output_dir}")
+        except Exception as e:
+            logger.warning(f"Failed to create {self.output_dir}, falling back to 'output': {e}")
+            self.output_dir = 'output'
+            os.makedirs(self.output_dir, exist_ok=True)
+
+        # Get duration from environment variable
         try:
-            self.duration_minutes = int(os.getenv('EXACT_DURATION', '30'))
-            logger.info(f"Using exact duration of {self.duration_minutes} minutes from test run")
+            self.duration_minutes = int(os.getenv('DURATION_TIME', '30'))
+            logger.info(f"Using duration of {self.duration_minutes} minutes")
         except ValueError as e:
             logger.error(f"Error parsing duration: {e}")
             self.duration_minutes = 30

get_pprof.py

@@ -92,17 +92,34 @@ def convert_to_speedscope(profile_data: Dict[str, Any], pod_name: str) -> Dict[s
 
 
 def main():
-    # Read EXACT_DURATION from environment (default to 30 minutes)
-    duration_minutes = int(os.getenv('EXACT_DURATION', '30'))
-    print(f"Using EXACT_DURATION={duration_minutes} minutes from environment")
+    # Read environment variables with defaults
+    duration_minutes = int(os.getenv('DURATION_TIME', '30'))
+    base_output_dir = os.getenv('OUTPUT_DIR', 'output')
+    print(f"Using DURATION_TIME={duration_minutes} minutes")
+    print(f"Using base output directory: {base_output_dir}")
 
     # Calculate time range
     until_time = datetime.now(timezone.utc)
     from_time = until_time - timedelta(minutes=duration_minutes)
 
-    # Create output directory
-    output_dir = Path("output/profiles")
-    output_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        # Try to create the base output directory
+        os.makedirs(base_output_dir, exist_ok=True)
+    except Exception as e:
+        print(f"Failed to create {base_output_dir}, falling back to 'output': {e}")
+        base_output_dir = 'output'
+        os.makedirs(base_output_dir, exist_ok=True)
+
+    # Create profiles subdirectory within the base output directory
+    output_dir = Path(base_output_dir) / "profiles"
+    try:
+        output_dir.mkdir(parents=True, exist_ok=True)
+        print(f"Successfully created profiles directory: {output_dir}")
+    except Exception as e:
+        print(f"Error creating profiles directory: {e}")
+        # Fallback to base output directory
+        output_dir = Path(base_output_dir)
+        print(f"Falling back to base directory: {output_dir}")
 
     # Get all node-agent pods
     pods = get_node_agent_pods()
@@ -130,9 +147,12 @@ def main():
                 output_file = output_dir / f"{pod_name}_{timestamp}.speedscope.json"
 
                 # Write to file
-                with open(output_file, 'w') as f:
-                    json.dump(speedscope_data, f)
-                print(f"Profile saved to {output_file}")
+                try:
+                    with open(output_file, 'w') as f:
+                        json.dump(speedscope_data, f)
+                    print(f"Profile saved to {output_file}")
+                except Exception as e:
+                    print(f"Error saving profile for {pod_name}: {e}")
             else:
                 print(f"Failed to convert profile data for {pod_name}")
         else: