os/kernel/sched: update cpuload measurement for smp

	- This commit calculates cpu load for each core
	  separately whereas previously it was only
	  calculating for one core. This gives the cpu load
          percentage of each thread for their respective cores.

	  It only prints threads that used the cpu.

	  The previous output when when we tash command as "cpuload" was :
	  TASH>> cpuload
	  Started CPU monitor with interval 5.
	  PID | Pri |    2s |
	  --------------------------------------------------
	    0 |   0 |  99.1 | Idle Task
  	    1 | 224 |   0.0 | hpwork
 	    2 | 113 |   0.0 | lpwork
	    3 | 120 |   0.0 | EHCI Monitor
	    4 | 110 |   0.0 | LWIP_TCP/IP
  	    5 | 250 |   0.0 | binary_manager
   	    7 | 220 |   0.0 | /dev/mtdblock2
  	    8 | 221 |   0.0 | msg_receiver
	    9 | 221 |   0.0 | multi_recv_nonblock
	   10 | 221 |   0.0 | multi_recv_block1
	   11 | 221 |   0.0 | multi_recv_block2
	   12 | 180 |   0.0 | /dev/mtdblock4
	   13 | 100 |   0.0 | uwork
	   14 | 125 |   0.9 | tash
	   15 | 100 |   0.0 | CPULoadMonitor
	  --------------------------------------------------

	  After this commit , output is :
	  Started CPU monitor with interval 5 sec.
	  Non-Zero CPU utilization trend (updated every 5s)
	  --------------------------------------------------
	  PID | Pri |  CPU0 |  CPU1 |  Avg  |
	  --------------------------------------------------
 	    0 |   0 |  99.9 |   0.0 |  42.8 | CPU0 IDLE
 	    1 |   0 |   0.0 | 100.0 |  57.2 | CPU1 IDLE
 	   11 | 200 |   0.0 |   0.1 |   0.1 | log_dump
 	   20 | 100 |   1.4 |   0.0 |   0.6 | CPULoadMonitor
	  --------------------------------------------------

apps/system/utils/README.md

@@ -126,7 +126,11 @@ Kernel Features -> Files and I/O -> Maximum number of file descriptors per task
 ```
 
 ## cpuload
-This command shows cpuload information per thread periodically. This has arguments which configure the printing information and cpuload daemon.
+This command shows cpuload information per thread periodically. 
+If there are more than one cpu cores in the system, it shows cpuload of thread in each core.
+Also It shows non-zero cpuload only.
+This has arguments which configure the printing information and cpuload daemon.
+
 ```bash
 TASH>>cpuload --help
 
@@ -140,27 +144,19 @@ Options:
  -i PRINT_INTERVAL     Show cpuload values every PRINT_INTERVAL(s)
  -n ITERATION_COUNT    Iterate showing cpuload values ITERATION_COUNT times
 
-TASH>> cpuload
-Started CPU monitor with interval 5.
-PID | Pri |    2s |
+TASH>>cpuload
+
+--------------------------------------------------
+Started CPU monitor with interval 5 sec.
+TASH>>Non-Zero CPU utilization trend (updated every 5s)
+PID | Pri |  CPU0 |  CPU1 |  Avg  |
 --------------------------------------------------
-  0 |   0 |  99.1 | Idle Task
-  1 | 224 |   0.0 | hpwork
-  2 | 113 |   0.0 | lpwork
-  3 | 120 |   0.0 | EHCI Monitor
-  4 | 110 |   0.0 | LWIP_TCP/IP
-  5 | 250 |   0.0 | binary_manager
-  7 | 220 |   0.0 | /dev/mtdblock2
-  8 | 221 |   0.0 | msg_receiver
-  9 | 221 |   0.0 | multi_recv_nonblock
- 10 | 221 |   0.0 | multi_recv_block1
- 11 | 221 |   0.0 | multi_recv_block2
- 12 | 180 |   0.0 | /dev/mtdblock4
- 13 | 100 |   0.0 | uwork
- 14 | 125 |   0.9 | tash
- 15 | 100 |   0.0 | CPULoadMonitor
+  0 |   0 |  99.9 |   0.0 |  48.5 | CPU0 IDLE
+  1 |   0 |   0.0 |  99.9 |  51.4 | CPU1 IDLE
+ 20 | 100 |   0.1 |   0.0 |   0.1 | CPULoadMonitor
 --------------------------------------------------
 
+
 TASH>> cpuload -s 10
 CPU monitor will started after 10s with interval 5.
 PID | Pri |  Snap ticks  |
@@ -183,6 +179,16 @@ PID | Pri |  Snap ticks  |
  * Snapshot interval : 10s (1000 ticks)
 --------------------------------------------------
 ```
+### How is the cpuload calculated
+We define a timeconstant (CONFIG_SCHED_CPULOAD_TIMECONSTANT), to keep a limit on total cpuload.
+We store "total ticks" and "ticks respective to a thread for specific core" after every systick interrupt.
+thread_n load on cpu_i = (thread_n ticks in cpu_i) / (total ticks in cpu_i)
+average thread_n load = (Sum of thread_n load in each cpu) / (Number of cpus)
+
+Note: Suppose timeconstant is 2 second. Then after every 2 second, we divide the number of ticks
+      on each thread_n load by 2 and also on each cpu_n load by 2. After this the process continues
+      until the next 2 seconds.
+
 ### How to Enable
 Enable *CONFIG_ENABLE_CPULOAD* to use this command on menuconfig as shown below:
 ```

apps/system/utils/utils_cpuload.c

@@ -109,6 +109,27 @@ static void cpuload_stop(void)
 	}
 }
 
+/* This function checks if the load of a thread is
+ * non zero or not. Basically, after calculating the load,
+ * we store the load info in the form of a string as follows:
+ * <cpu0_load>-<cpu1_load>-<avg_load> (for 2 core case)
+ * So for zero load case, we check if avg_load is 0.0 or not.
+ * Note: we compare with "-0.0" because if avg_load is 90.0, then
+ * also we will get true if we compare with 0.0 */
+static bool has_cpuload(char *load_info)
+{
+#ifdef CONFIG_SMP
+	int len = strlen(load_info);
+	if (strcmp(&load_info[len - 4], "-0.0") == 0) {
+#else
+	if (strcmp(load_info, "0.0") == 0) {
+#endif
+		return false;
+	}
+
+	return true;
+}
+
 static void cpuload_print_pid_value(char *buf, void *arg)
 {
 	int i;
@@ -123,6 +144,14 @@ static void cpuload_print_pid_value(char *buf, void *arg)
 		stat_info[i] = strtok_r(stat_info[i], " ", &stat_info[i + 1]);
 	}
 
+#ifdef CONFIG_SCHED_MULTI_CPULOAD
+	if (!(has_cpuload(stat_info[PROC_STAT_CPULOAD_SHORT]) || has_cpuload(stat_info[PROC_STAT_CPULOAD_MID]) || has_cpuload(stat_info[PROC_STAT_CPULOAD_LONG]))) {
+#else 
+	if (!has_cpuload(stat_info[PROC_STAT_CPULOAD])) {
+#endif
+		return;
+	}
+
 	printf("%3s | %3s |", stat_info[PROC_STAT_PID], stat_info[PROC_STAT_PRIORITY]);
 	if (cpuload_mode == CPULOAD_SNAPSHOT) {
 		pid = atoi(stat_info[PROC_STAT_PID]);
@@ -136,14 +165,35 @@ static void cpuload_print_pid_value(char *buf, void *arg)
 		printf(" %3d(%4.1f) |", cpuload_pidhash[pid_hash].count, load_ratio);
 	} else {
 #ifdef CONFIG_SCHED_MULTI_CPULOAD
-		printf(" %5s | %5s | %5s |", stat_info[PROC_STAT_CPULOAD_SHORT], stat_info[PROC_STAT_CPULOAD_MID], stat_info[PROC_STAT_CPULOAD_LONG]);
+		for (i = PROC_STAT_CPULOAD_SHORT; i <= PROC_STAT_CPULOAD_LONG; i++) {
+#ifdef CONFIG_SMP
+			char * avgload[CONFIG_SMP_NCPUS + 1];
+			avgload[0] = stat_info[i];
+			for (int j = 0; j < CONFIG_SMP_NCPUS + 1; j++) {
+				avgload[j] = strtok_r(avgload[j], "-", &avgload[j + 1]);
+				printf(" %5s |", avgload[j]);
+			}
+#else
+			printf(" %5s |", stat_info[i]);
+#endif
+		}
+#else
+#ifdef CONFIG_SMP
+		char * avgload[CONFIG_SMP_NCPUS + 1];
+		avgload[0] = stat_info[PROC_STAT_CPULOAD];
+		for (i = 0; i < CONFIG_SMP_NCPUS + 1; i++) {
+			avgload[i] = strtok_r(avgload[i], "-", &avgload[i + 1]);
+			printf(" %5s |", avgload[i]);
+		}
 #else
 		printf(" %5s |", stat_info[PROC_STAT_CPULOAD]);
+#endif
 #endif
 	}
 #if (CONFIG_TASK_NAME_SIZE > 0)
 	printf(" %s", stat_info[PROC_STAT_NAME]);
 #endif
+	printf("\n");
 }
 
 static int cpuload_read_proc(FAR struct dirent *entryp, FAR void *arg)
@@ -159,19 +209,34 @@ static int cpuload_read_proc(FAR struct dirent *entryp, FAR void *arg)
 		printf("Failed to read %s\n", filepath);
 		return ERROR;
 	}
-	printf("\n");
 	return OK;
 }
 
 static void cpuload_print_normal(void)
 {
 	/* Print titles */
+	printf("\n--------------------------------------------------\n");
+	printf("Non-Zero CPU utilization trend (updated every %ds)\n", cpuload_interval);
+#ifdef CONFIG_SCHED_MULTI_CPULOAD
+	printf("Multi cpuload mode is enabled with time constants %ds, %ds, %ds (used to average out cpuload after every time constants)\n", CONFIG_SCHED_CPULOAD_TIMECONSTANT_SHORT, CONFIG_SCHED_CPULOAD_TIMECONSTANT_MID, CONFIG_SCHED_CPULOAD_TIMECONSTANT_LONG);
+#endif
 	printf("PID | Pri |");
+
 #ifdef CONFIG_SCHED_MULTI_CPULOAD
-	printf("%5ds | %4ds | %4ds |", CONFIG_SCHED_CPULOAD_TIMECONSTANT_SHORT, CONFIG_SCHED_CPULOAD_TIMECONSTANT_MID, CONFIG_SCHED_CPULOAD_TIMECONSTANT_LONG);
+	for (int i = PROC_STAT_CPULOAD_SHORT; i <= PROC_STAT_CPULOAD_LONG; i++) {
+#endif
+#ifdef CONFIG_SMP
+		for (int j = 0; j < CONFIG_SMP_NCPUS; j++) {
+			printf("  CPU%d |", j);
+		}
+#endif
+#ifndef CONFIG_SCHED_MULTI_CPULOAD
+		printf("  Avg  |");
 #else
-	printf("%5ds |", CONFIG_SCHED_CPULOAD_TIMECONSTANT);
+		printf("  Avg%d |", i - PROC_STAT_CPULOAD_SHORT + 1);
+	}
 #endif
+
 	printf("\n--------------------------------------------------\n");
 
 	/* Print cpuload for each task */
@@ -325,9 +390,9 @@ static int cpuload_start(void)
 	pthread_setname_np(cpuloadmon, "CPULoadMonitor");
 
 	if (cpuload_mode == CPULOAD_SNAPSHOT) {
-		printf("CPU monitor will started after %ds with interval %d.\n", cpuload_snapintval, cpuload_interval);
+		printf("CPU monitor will be started after %ds with interval %d sec.\n", cpuload_snapintval, cpuload_interval);
 	} else {
-		printf("Started CPU monitor with interval %d.\n", cpuload_interval);
+		printf("Started CPU monitor with interval %d sec.\n", cpuload_interval);
 	}
 
 	return OK;

os/crypto/random_pool.c

@@ -245,7 +245,12 @@ static void getentropy(FAR blake2s_state *S)
 	tmp <<= 27;
 #ifdef CONFIG_SCHED_CPULOAD
 	clock_cpuload(getpid(), 0, &load);
-	tmp += load.total ^ ROTL_32(load.active, 23);
+	uint32 total = 0, active = 0;
+	for (int cpu = 0; cpu < CONFIG_SMP_NCPUS; cpu++) {
+		total += load.total[cpu];
+		active += load.active[cpu];
+	}
+	tmp += total ^ ROTL_32(active, 23);
 #endif
 	add_sw_randomness(tmp);
 

os/fs/procfs/fs_procfscpuload.c

@@ -242,6 +242,8 @@ static ssize_t cpuload_read(FAR struct file *filep, FAR char *buffer, size_t buf
 		struct cpuload_s cpuload;
 		uint32_t intpart;
 		uint32_t fracpart;
+		uint32_t total_cpuload = 0;
+		uint32_t total_active = 0;
 
 		/* Sample the counts for the IDLE thread.  clock_cpuload should only
 		 * fail if the PID is not valid.  This, however, should never happen
@@ -253,15 +255,19 @@ static ssize_t cpuload_read(FAR struct file *filep, FAR char *buffer, size_t buf
 		for (cpuload_idx = 0; cpuload_idx < SCHED_NCPULOAD; cpuload_idx++) {
 
 			DEBUGVERIFY(clock_cpuload(0, cpuload_idx, &cpuload));
+			for (int cpu = 0; cpu < CONFIG_SMP_NCPUS; cpu++) {
+				total_cpuload += cpuload.total[cpu];
+				total_active += cpuload.active[cpu];
+			}
 
 			/* On the simulator, you may hit cpuload.total == 0, but probably never on
 			 * real hardware.
 			 */
 
-			if (cpuload.total > 0) {
+			if (total_cpuload > 0) {
 				uint32_t tmp;
 
-				tmp = 1000 - (1000 * cpuload.active) / cpuload.total;
+				tmp = 1000 - (1000 * total_active) / total_cpuload;
 				intpart = tmp / 10;
 				fracpart = tmp - 10 * intpart;
 			} else {

os/fs/procfs/fs_procfsproc.c

@@ -371,6 +371,8 @@ static ssize_t proc_entry_stat(FAR struct proc_file_s *procfile, FAR struct tcb_
 	int cpuload_idx;
 	double load_value;
 	struct cpuload_s cpuload;
+	uint32_t total_cpuload = 0;
+	uint32_t total_active = 0;
 #endif
 
 	remaining = buflen;
@@ -407,27 +409,52 @@ static ssize_t proc_entry_stat(FAR struct proc_file_s *procfile, FAR struct tcb_
 #else
 	ppid = -1;
 #endif
-	linesize = snprintf(procfile->line, STATUS_LINELEN, "%d %d %d %d %d %d %d %d %d", tcb->pid, ppid, tcb->sched_priority, tcb->flags, tcb->task_state, tcb->adj_stack_size, peak_stack, curr_heap, peak_heap);
+	linesize = snprintf(procfile->line, STATUS_LINELEN, "%d %d %d %d %d %d %d %d %d ", tcb->pid, ppid, tcb->sched_priority, tcb->flags, tcb->task_state, tcb->adj_stack_size, peak_stack, curr_heap, peak_heap);
 	copysize = procfs_memcpy(procfile->line, linesize, buffer, buflen, &offset);
 	totalsize += copysize;
 
 #ifdef CONFIG_SCHED_CPULOAD
 	for (cpuload_idx = 0; cpuload_idx < SCHED_NCPULOAD; cpuload_idx++) {
-		buffer += copysize;
-		remaining -= copysize;
-		if (totalsize >= buflen) {
-			return totalsize;
-		}
-
 		/* Get cpuload measurement data */
 		(void)clock_cpuload(procfile->pid, cpuload_idx, &cpuload);
-		if (cpuload.total > 0) {
-			load_value = (double)(100 * cpuload.active) / (double)cpuload.total;
+
+		for (int cpu = 0; cpu < CONFIG_SMP_NCPUS; cpu++) {
+
+			if (cpuload.total[cpu] > 0) {
+				load_value = (double)(100 * cpuload.active[cpu]) / (double)cpuload.total[cpu];
+				total_cpuload += cpuload.total[cpu];
+				total_active += cpuload.active[cpu];
+			} else {
+				load_value = 0.0f;
+			}
+
+#ifdef CONFIG_SMP
+			buffer += copysize;
+			remaining -= copysize;
+			if (totalsize >= buflen) {
+				return totalsize;
+			}
+
+			linesize = snprintf(procfile->line, STATUS_LINELEN, "%.1f-", load_value);
+			copysize = procfs_memcpy(procfile->line, linesize, buffer, remaining, &offset);
+			totalsize += copysize;
+#endif
+
+		}
+
+		if (total_cpuload > 0) {
+			load_value = (double)(100 * total_active) / (double)total_cpuload;
 		} else {
 			load_value = 0.0f;
 		}
 
-		linesize = snprintf(procfile->line, STATUS_LINELEN, " %.1f", load_value);
+		buffer += copysize;
+		remaining -= copysize;
+		if (totalsize >= buflen) {
+			return totalsize;
+		}
+
+		linesize = snprintf(procfile->line, STATUS_LINELEN, "%.1f ", load_value);
 		copysize = procfs_memcpy(procfile->line, linesize, buffer, remaining, &offset);
 		totalsize += copysize;
 	}
@@ -654,6 +681,8 @@ static ssize_t proc_entry_loadavg(FAR struct proc_file_s *procfile, FAR struct t
 	size_t copysize;
 	size_t totalsize;
 	int cpuload_idx;
+	uint32_t total_cpuload = 0;
+	uint32_t total_active = 0;
 
 	/* Sample the counts for the thread.  clock_cpuload should only fail if
 	 * the PID is not valid.  This could happen if the thread exited sometime
@@ -666,14 +695,18 @@ static ssize_t proc_entry_loadavg(FAR struct proc_file_s *procfile, FAR struct t
 
 		(void)clock_cpuload(procfile->pid, cpuload_idx, &cpuload);
 
+		for (int cpu = 0; cpu < CONFIG_SMP_NCPUS; cpu++) {
+			total_cpuload += cpuload.total[cpu];
+			total_active += cpuload.active[cpu];
+		}
+
 		/* On the simulator, you may hit cpuload.total == 0, but probably never on
 		 * real hardware.
 		 */
 
-		if (cpuload.total > 0) {
+		if (total_cpuload > 0) {
 			uint32_t tmp;
-
-			tmp = (1000 * cpuload.active) / cpuload.total;
+			tmp = (1000 * total_active) / total_cpuload;
 			intpart = tmp / 10;
 			fracpart = tmp - 10 * intpart;
 		} else {

os/include/tinyara/clock.h

@@ -200,8 +200,8 @@
 
 #ifdef CONFIG_SCHED_CPULOAD
 struct cpuload_s {
-	volatile uint32_t total;	/* Total number of clock ticks */
-	volatile uint32_t active;	/* Number of ticks while this thread was active */
+	volatile uint32_t total[CONFIG_SMP_NCPUS];   /* Total number of clock ticks per cpu */
+	volatile uint32_t active[CONFIG_SMP_NCPUS];  /* Number of ticks while this thread was active on the specific cpu */
 };
 
 #ifdef CONFIG_SCHED_MULTI_CPULOAD

os/kernel/sched/sched.h

@@ -137,7 +137,8 @@ struct pidhash_s {
 	FAR struct tcb_s *tcb;		/* TCB assigned to this PID */
 	pid_t pid;					/* The full PID value */
 #ifdef CONFIG_SCHED_CPULOAD
-	uint32_t ticks[SCHED_NCPULOAD];				/* Number of ticks on this thread */
+	uint32_t ticks[CONFIG_SMP_NCPUS][SCHED_NCPULOAD];     /* Number of ticks of thread in specific cpu */
+
 #endif
 };