diff --git a/csm_web/frontend/src/components/App.tsx b/csm_web/frontend/src/components/App.tsx index 6799ce7c..1c321f7a 100644 --- a/csm_web/frontend/src/components/App.tsx +++ b/csm_web/frontend/src/components/App.tsx @@ -8,6 +8,7 @@ import { emptyRoles, Roles } from "../utils/user"; import CourseMenu from "./CourseMenu"; import Home from "./Home"; import Policies from "./Policies"; +import { DataExport } from "./data_export/DataExport"; import { EnrollmentMatcher } from "./enrollment_automation/EnrollmentMatcher"; import { Resources } from "./resource_aggregation/Resources"; import Section from "./section/Section"; @@ -39,6 +40,7 @@ const App = () => { } /> } /> } /> + } /> } /> diff --git a/csm_web/frontend/src/components/AutoGrid.tsx b/csm_web/frontend/src/components/AutoGrid.tsx new file mode 100644 index 00000000..43f0bfb1 --- /dev/null +++ b/csm_web/frontend/src/components/AutoGrid.tsx @@ -0,0 +1,48 @@ +import React from "react"; + +import "../css/base/autogrid.scss"; + +interface AutoColumnsProps { + children?: React.ReactNode[]; +} + +/** + * Automatically format children in balanced columns. + */ +export const AutoGrid = ({ children }: AutoColumnsProps) => { + const gridSize = Math.ceil(Math.sqrt(children?.length ?? 0)); + + if (children == null) { + return null; + } + + const raw_table: React.ReactNode[][] = []; + children.forEach((item, idx) => { + if (idx % gridSize == 0) { + raw_table.push([item]); + } else { + raw_table[raw_table.length - 1].push(item); + } + }); + + // transpose table + const table = raw_table[0].map((_, colIndex) => raw_table.map(row => row[colIndex])); + + return ( +
+ + + {table.map((row, rowIdx) => ( + + {row.map((item, itemIdx) => ( + + ))} + + ))} + +
+ {item} +
+
+ ); +}; diff --git a/csm_web/frontend/src/components/Home.tsx b/csm_web/frontend/src/components/Home.tsx index b8f0298f..da7873e0 100644 --- a/csm_web/frontend/src/components/Home.tsx +++ b/csm_web/frontend/src/components/Home.tsx @@ -8,6 +8,7 @@ import { useCourses } from "../utils/queries/courses"; import { Profile, Course, Role } from "../utils/types"; import LoadingSpinner from "./LoadingSpinner"; +import FileExport from "../../static/frontend/img/file-export.svg"; import PlusIcon from "../../static/frontend/img/plus.svg"; import scssColors from "../css/base/colors-export.module.scss"; @@ -17,6 +18,7 @@ const Home = () => { const { data: courses, isSuccess: coursesLoaded, isError: coursesLoadError } = useCourses(); let content = null; + let headingRight = null; if (profilesLoaded && coursesLoaded) { // loaded, no error const coursesById: Map = new Map(); @@ -52,6 +54,17 @@ const Home = () => { })} ); + + const isCoordinator = profiles!.some(profile => profile.role === Role.COORDINATOR); + + if (isCoordinator) { + headingRight = ( + + + Export + + ); + } } else if (profilesLoadError) { // error during load content =

Profiles not found

; @@ -66,11 +79,14 @@ const Home = () => { return (
-

My courses

- - - Add Course - +
+

My courses

+ + + Add Course + +
+
{headingRight}
{content}
diff --git a/csm_web/frontend/src/components/course/Course.tsx b/csm_web/frontend/src/components/course/Course.tsx index a46a3f14..e1d20cbc 100644 --- a/csm_web/frontend/src/components/course/Course.tsx +++ b/csm_web/frontend/src/components/course/Course.tsx @@ -6,7 +6,6 @@ import { useCourseSections } from "../../utils/queries/courses"; import { Course as CourseType } from "../../utils/types"; import LoadingSpinner from "../LoadingSpinner"; import { CreateSectionModal } from "./CreateSectionModal"; -import { DataExportModal } from "./DataExportModal"; import { SectionCard } from "./SectionCard"; import { SettingsModal } from "./SettingsModal"; import { WhitelistModal } from "./WhitelistModal"; @@ -27,7 +26,6 @@ const DAY_OF_WEEK_ABREVIATIONS: { [day: string]: string } = Object.freeze({ }); const COURSE_MODAL_TYPE = Object.freeze({ - exportData: "csv", createSection: "mksec", whitelist: "whitelist", settings: "settings" @@ -83,9 +81,7 @@ const Course = ({ courses, priorityEnrollment, enrollmentTimes }: CourseProps): * Render the currently chosen modal. */ const renderModal = (): React.ReactElement | null => { - if (whichModal == COURSE_MODAL_TYPE.exportData) { - return setShowModal(false)} />; - } else if (whichModal == COURSE_MODAL_TYPE.createSection) { + if (whichModal == COURSE_MODAL_TYPE.createSection) { return ( Create Section - - - - - ); -}; diff --git a/csm_web/frontend/src/components/data_export/DataExport.tsx b/csm_web/frontend/src/components/data_export/DataExport.tsx new file mode 100644 index 00000000..5cadff82 --- /dev/null +++ b/csm_web/frontend/src/components/data_export/DataExport.tsx @@ -0,0 +1,44 @@ +import React, { useState } from "react"; +import { useProfiles } from "../../utils/queries/base"; +import { Role } from "../../utils/types"; +import LoadingSpinner from "../LoadingSpinner"; +import { ExportType } from "./DataExportTypes"; +import { ExportPage } from "./ExportPage"; +import { ExportSelector } from "./ExportSelector"; + +export const DataExport = () => { + const [dataExportType, setDataExportType] = useState(null); + const { data: profiles, isSuccess: profilesLoaded, isError: profilesError } = useProfiles(); + + if (profilesError) { + return Error loading user profiles.; + } else if (!profilesLoaded) { + return ; + } else if (profilesLoaded && !profiles.some(profile => profile.role === Role.COORDINATOR)) { + return Permission denied; you are not a coordinator for any course.; + } + + return ( +
+
+

Export Data

+
+
+
+ { + setDataExportType(exportType); + }} + /> +
+
+ {dataExportType === null ? ( +
Select export type to start.
+ ) : ( + + )} +
+
+
+ ); +}; diff --git a/csm_web/frontend/src/components/data_export/DataExportTypes.tsx b/csm_web/frontend/src/components/data_export/DataExportTypes.tsx new file mode 100644 index 00000000..d117cd21 --- /dev/null +++ b/csm_web/frontend/src/components/data_export/DataExportTypes.tsx @@ -0,0 +1,87 @@ +/** + * Enum for all the various data export types + */ +export enum ExportType { + STUDENT_DATA = "STUDENT_DATA", + ATTENDANCE_DATA = "ATTENDANCE_DATA", + SECTION_DATA = "SECTION_DATA", + COURSE_DATA = "COURSE_DATA" +} + +/** + * Object for displaying export types in the UI + */ +export const EXPORT_TYPE_DATA = new Map([ + [ExportType.STUDENT_DATA, "Student"], + [ExportType.ATTENDANCE_DATA, "Attendance"], + [ExportType.SECTION_DATA, "Section"], + [ExportType.COURSE_DATA, "Course"] +]); + +export const EXPORT_COLUMNS: { + [exportType in ExportType]: { + required: { [key: string]: string }; + optional: { [key: string]: string }; + }; +} = { + [ExportType.ATTENDANCE_DATA]: { + required: { + student_email: "Student email", + student_name: "Student name", + attendance_data: "Attendance data" + }, + optional: { + course_name: "Course name", + active: "Active", + section_id: "Section ID", + mentor_name: "Mentor name", + mentor_email: "Mentor email", + num_present: "Present attendance count", + num_excused: "Excused absence count", + num_unexcused: "Unexcused absence count" + } + }, + [ExportType.COURSE_DATA]: { + required: { + course_name: "Course name" + }, + optional: { + course_id: "Course ID", + description: "Course description", + num_sections: "Section count", + num_students: "Student count", + num_mentors: "Mentor count" + } + }, + [ExportType.SECTION_DATA]: { + required: { + mentor_name: "Mentor name", + mentor_email: "Mentor email" + }, + optional: { + course_name: "Course name", + section_id: "Section ID", + section_times: "Section times", + section_description: "Section description", + num_students: "Student count", + capacity: "Capacity" + } + }, + [ExportType.STUDENT_DATA]: { + required: { + student_email: "Student email", + student_name: "Student name" + }, + optional: { + course_name: "Course name", + active: "Active", + mentor_name: "Mentor name", + mentor_email: "Mentor email", + section_id: "Section ID", + section_times: "Section times", + num_present: "Present attendance count", + num_excused: "Excused absence count", + num_unexcused: "Unexcused absence count" + } + } +}; diff --git a/csm_web/frontend/src/components/data_export/ExportPage.tsx b/csm_web/frontend/src/components/data_export/ExportPage.tsx new file mode 100644 index 00000000..808b6c85 --- /dev/null +++ b/csm_web/frontend/src/components/data_export/ExportPage.tsx @@ -0,0 +1,258 @@ +import React, { useEffect, useState } from "react"; +import { useProfiles } from "../../utils/queries/base"; +import { useDataExportMutation, useDataExportPreviewMutation } from "../../utils/queries/export"; +import { Role } from "../../utils/types"; +import { AutoGrid } from "../AutoGrid"; +import LoadingSpinner from "../LoadingSpinner"; +import { Tooltip } from "../Tooltip"; +import { ExportType, EXPORT_COLUMNS } from "./DataExportTypes"; + +import RefreshIcon from "../../../static/frontend/img/refresh.svg"; + +interface ExportPageProps { + dataExportType: ExportType; +} + +export const ExportPage = ({ dataExportType }: ExportPageProps) => { + const { data: profiles, isSuccess: profilesLoaded, isError: profilesError } = useProfiles(); + const [includedCourses, setIncludedCourses] = useState([]); + const [includedFields, setIncludedFields] = useState( + Array.from(Object.keys(EXPORT_COLUMNS[dataExportType].optional)) + ); + + const dataExportMutation = useDataExportMutation(); + + useEffect(() => { + if (profiles != null && profilesLoaded) { + const coordinatorProfiles = profiles.filter(profile => profile.role === Role.COORDINATOR); + setIncludedCourses(coordinatorProfiles.map(profile => profile.courseId)); + } + }, [profilesLoaded, profiles]); + + useEffect(() => { + setIncludedFields(Array.from(Object.keys(EXPORT_COLUMNS[dataExportType].optional))); + }, [dataExportType]); + + if (profilesError) { + return

Profiles not found

; + } else if (!profilesLoaded) { + return ; + } + + const coordinatorProfiles = profiles.filter(profile => profile.role === Role.COORDINATOR); + + const courseSelection = ( +
+

Select Courses

+
+ + {coordinatorProfiles + .sort((profileA, profileB) => profileA.course.localeCompare(profileB.course)) + .map(profile => ( + + ))} + +
+
+ ); + + const columnFields = EXPORT_COLUMNS[dataExportType]; + const requiredInputs = Object.entries(columnFields.required).map(([key, description]) => ({ + key, + description, + disabled: true + })); + const optionalInputs = Object.entries(columnFields.optional).map(([key, description]) => ({ + key, + description, + disabled: false + })); + const columnInputs = requiredInputs.concat(optionalInputs).map(({ key, description, disabled }) => ( + + )); + + const columnSelection = ( +
+

Select Fields

+
+ {columnInputs} +
+
+ ); + + /** + * Download the data; open a new page with the data + */ + const downloadData = () => { + if (includedCourses.length === 0) { + // no data to download + return; + } + + dataExportMutation.mutate({ + courses: includedCourses, + fields: includedFields, + type: dataExportType + }); + }; + + return ( +
+
+
{courseSelection}
+
{columnSelection}
+
+ +
+ +
+
+ ); +}; + +const PREVIEW_OPTIONS = [5, 10, 25, 50]; + +interface ExportPagePreviewProps { + courses: number[]; + fields: string[]; + exportType: ExportType; +} + +/** + * Preview of the exported data + */ +const ExportPagePreview = ({ exportType, courses, fields }: ExportPagePreviewProps) => { + const dataExportPreviewMutation = useDataExportPreviewMutation(); + const [preview, setPreview] = useState(10); + const [data, setData] = useState([]); + + // automatically refresh on change + useEffect(() => { + refreshPreview(); + }, [courses, fields, preview]); + + const refreshPreview = () => { + dataExportPreviewMutation.mutate( + { + courses: courses, + fields: fields, + type: exportType, + preview: preview + }, + { + onSuccess: dataPreview => { + setData(dataPreview); + } + } + ); + }; + + const handlePreviewSelect = (e: React.ChangeEvent) => { + const selectedValue = parseInt(e.target.value); + setPreview(selectedValue); + }; + + let dataTable = null; + + if (data?.length > 1) { + // if has header and at least one row of content, display it + dataTable = ( + + + + {data?.length > 0 && + data[0].map((cell, cellIdx) => ( + + ))} + + + + {data.map( + (row, rowIdx) => + rowIdx > 0 && ( + + {row.map((cell, cellIdx) => ( + + ))} + + ) + )} + {data?.length >= preview && ( + + + + )} + +
+ {cell} +
+ {cell} +
+ (More rows clipped) +
+ ); + } else { + // not enough data + dataTable = Preview query returned no data.; + } + + return ( +
+

Preview

+
+
+ Rows: + +
+ } + placement="right" + > + Refresh + +
+
+
{dataTable}
+
+
+ ); +}; diff --git a/csm_web/frontend/src/components/data_export/ExportSelector.tsx b/csm_web/frontend/src/components/data_export/ExportSelector.tsx new file mode 100644 index 00000000..e05e215c --- /dev/null +++ b/csm_web/frontend/src/components/data_export/ExportSelector.tsx @@ -0,0 +1,40 @@ +import React, { useState } from "react"; + +import { ExportType, EXPORT_TYPE_DATA } from "./DataExportTypes"; + +import "../../css/data-export.scss"; + +interface ExportSelectorProps { + onSelect: (exportType: ExportType) => void; +} + +/** + * Component for selecting the courses to include in the export, + * along with the export data config selection. + */ +export const ExportSelector = ({ onSelect }: ExportSelectorProps) => { + const [dataExportType, setDataExportType] = useState(null); + + const handleSelect = (exportType: ExportType) => { + onSelect(exportType); + setDataExportType(exportType); + }; + + return ( +
+
+ {Array.from(EXPORT_TYPE_DATA.entries()) + .sort() + .map(([exportType, description]) => ( +
handleSelect(exportType)} + > + {description} +
+ ))} +
+
+ ); +}; diff --git a/csm_web/frontend/src/css/base/autogrid.scss b/csm_web/frontend/src/css/base/autogrid.scss new file mode 100644 index 00000000..910ccd6e --- /dev/null +++ b/csm_web/frontend/src/css/base/autogrid.scss @@ -0,0 +1,5 @@ +@use "variables" as *; + +.auto-grid-item { + padding: 0 8px; +} diff --git a/csm_web/frontend/src/css/data-export.scss b/csm_web/frontend/src/css/data-export.scss new file mode 100644 index 00000000..cf7dcb04 --- /dev/null +++ b/csm_web/frontend/src/css/data-export.scss @@ -0,0 +1,204 @@ +/* Data export styles */ +@use "base/variables" as *; + +.data-export-container { + display: flex; + flex-direction: column; + align-items: stretch; + justify-content: center; + + padding-right: 16px; +} + +.data-export-body { + display: flex; + flex-direction: row; + + gap: 40px; +} + +.data-export-sidebar { + min-width: 150px; +} + +.data-export-content { + flex: 1; +} + +.export-selector-container { + display: flex; + flex-direction: column; + gap: 50px; + + padding-left: 24px; + margin-top: 16px; +} + +.export-selector-footer { + display: flex; + align-items: center; + justify-content: center; +} + +.export-page-sidebar-container { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; +} + +.export-page-sidebar-title { + margin-bottom: 8px; +} + +.export-selector-data-type-options { + display: flex; + flex-direction: column; + gap: 24px; + align-items: flex-start; + justify-content: center; + + width: fit-content; +} + +.export-page-courses-options { + width: fit-content; + max-height: 25%; +} + +.export-selector-data-type-label { + display: block; + width: 100%; + + font-size: 1.1rem; + + white-space: nowrap; + + cursor: pointer; + user-select: none; + + &.active { + color: $csm-green-darkened; + } +} + +.export-page-container { + display: flex; + flex-direction: column; + gap: 16px; + align-items: stretch; +} + +.export-page-config { + display: flex; + flex-flow: row wrap; + gap: 8px; +} + +.export-page-sidebar { + min-width: 650px; + padding: 16px 0; + background-color: #f3f3f3; + border-radius: 12px; +} + +.export-page-sidebar.sidebar-left { + flex: 1; +} + +.export-page-sidebar.sidebar-right { + flex: 2; +} + +.export-page-header { + display: flex; + gap: 16px; + align-items: center; + justify-content: flex-start; +} + +.export-page-footer { + display: flex; + flex-direction: row; + align-items: center; + justify-content: flex-start; +} + +.export-page-preview { + flex: 2; +} + +.export-page-preview-container { + display: flex; + flex-direction: column; + gap: 4px; +} + +.export-page-preview-title { + // make margin smaller + margin-bottom: 8px; +} + +.export-preview-header { + display: flex; + flex-direction: row; + gap: 8px; + align-items: center; + + margin-bottom: 8px; +} + +.export-preview-select { + width: 75px !important; +} + +.export-preview-icon { + color: black; + cursor: pointer; +} + +.export-preview-icon:hover { + color: #222; +} + +.export-preview-refresh-tooltip-container { + position: relative; + + .tooltip-body { + // offset tooltip slightly + margin-left: 8px; + } +} + +.export-preview-table-container { + max-width: 80vw; + overflow-x: auto; +} + +.export-page-preview-wrapper { + padding: 0 12px; +} + +.export-preview-table { + border-collapse: collapse; +} + +.export-preview-table-item, +.export-preview-table-header-item { + padding: 4px; + + white-space: nowrap; + border: 1px #aaa solid; +} + +.export-preview-table-header-item { + text-align: left; +} + +.export-preview-table-more-row-item { + padding: 8px 0; + color: #888; + + column-span: all; + text-align: left; +} diff --git a/csm_web/frontend/src/css/home.scss b/csm_web/frontend/src/css/home.scss index 6fd58c0d..2b515ee8 100644 --- a/csm_web/frontend/src/css/home.scss +++ b/csm_web/frontend/src/css/home.scss @@ -70,11 +70,23 @@ flex-wrap: wrap; align-items: center; justify-content: space-between; - width: 50%; - max-width: 300px; margin-bottom: 40px; } +#home-courses-heading-left { + display: flex; + flex: 1; + flex-direction: row; + gap: 20px; + align-items: center; + justify-content: flex-start; +} + +#home-courses-heading-right { + display: flex; + flex-direction: row; +} + .section-link { padding: 10px; font-size: 14px; diff --git a/csm_web/frontend/src/utils/api.tsx b/csm_web/frontend/src/utils/api.tsx index e7530611..2e8fbbdc 100644 --- a/csm_web/frontend/src/utils/api.tsx +++ b/csm_web/frontend/src/utils/api.tsx @@ -21,13 +21,21 @@ export function normalizeEndpoint(endpoint: string) { return `/api/${endpoint}`; } -export function fetchWithMethod(endpoint: string, method: string, data: any = {}, isFormData = false) { +export function fetchWithMethod( + endpoint: string, + method: string, + data: any = {}, + isFormData = false, + queryParams: URLSearchParams | null = null +) { if (!Object.prototype.hasOwnProperty.call(HTTP_METHODS, method)) { // check that method choice is valid throw new Error("HTTP method must be one of: POST, GET, PUT, PATCH, or DELETE"); } + const normalizedEndpoint = endpointWithQueryParams(normalizeEndpoint(endpoint), queryParams); + if (isFormData) { - return fetch(normalizeEndpoint(endpoint), { + return fetch(normalizedEndpoint, { method: method, credentials: "same-origin", headers: { @@ -36,7 +44,7 @@ export function fetchWithMethod(endpoint: string, method: string, data: any = {} body: data }); } - return fetch(normalizeEndpoint(endpoint), { + return fetch(normalizedEndpoint, { method: method, credentials: "same-origin", headers: { @@ -51,11 +59,27 @@ export function fetchWithMethod(endpoint: string, method: string, data: any = {} /** * Fetch data from normalized endpoint. */ -export async function fetchNormalized(endpoint: string) { - return await fetch(normalizeEndpoint(endpoint), { credentials: "same-origin" }); +export async function fetchNormalized(endpoint: string, queryParams: URLSearchParams | null = null) { + const normalizedEndpoint = normalizeEndpoint(endpoint); + return await fetch(endpointWithQueryParams(normalizedEndpoint, queryParams), { credentials: "same-origin" }); } -export async function fetchJSON(endpoint: string) { - const response = await fetch(normalizeEndpoint(endpoint), { credentials: "same-origin" }); +export async function fetchJSON(endpoint: string, queryParams: URLSearchParams | null = null) { + const normalizedEndpoint = normalizeEndpoint(endpoint); + const response = await fetch(endpointWithQueryParams(normalizedEndpoint, queryParams), { + credentials: "same-origin" + }); return await response.json(); } + +/** + * Add query parameters to the endpoint, if necessary. + * If no query parameters, then the endpoint is returned unchanged. + */ +export function endpointWithQueryParams(endpoint: string, queryParams: URLSearchParams | null = null) { + if (queryParams !== null) { + return `${endpoint}?${queryParams}`; + } else { + return endpoint; + } +} diff --git a/csm_web/frontend/src/utils/queries/export.tsx b/csm_web/frontend/src/utils/queries/export.tsx new file mode 100644 index 00000000..1407d468 --- /dev/null +++ b/csm_web/frontend/src/utils/queries/export.tsx @@ -0,0 +1,93 @@ +import { useMutation, UseMutationResult } from "@tanstack/react-query"; +import { parse as csv_parse } from "csv-parse/browser/esm/sync"; + +import { ExportType } from "../../components/data_export/DataExportTypes"; +import { endpointWithQueryParams, fetchNormalized, normalizeEndpoint } from "../api"; +import { handleError, handlePermissionsError, handleRetry, ServerError } from "./helpers"; + +interface DataExportPreviewMutationRequest { + courses: number[]; + fields: string[]; + type: ExportType; + preview: number; +} + +/** + * Mutation for fetching export data for preview. + * Returns a table containing the CSV contents. + */ +export const useDataExportPreviewMutation = (): UseMutationResult< + string[][], + ServerError, + DataExportPreviewMutationRequest +> => { + const mutationResult = useMutation( + async (body: DataExportPreviewMutationRequest) => { + if (body.courses.length === 0) { + // if no courses specified, then return an empty table; + // no request is needed + return [[]]; + } + + const response = await fetchNormalized( + "/export", + new URLSearchParams({ + courses: body.courses.join(","), + fields: body.fields.join(","), + type: body.type, + preview: body.preview.toString() + }) + ); + + if (response.ok) { + const content = await response.text(); + // format content into a table + const table = csv_parse(content); + return table; + } else { + handlePermissionsError(response.status); + throw new ServerError( + `Failed to fetch preview; type ${body.type}, courses ${body.courses}, fields ${body.fields}` + ); + } + }, + { + retry: handleRetry + } + ); + + handleError(mutationResult); + return mutationResult; +}; + +interface DataExportMutationRequest { + courses: number[]; + fields: string[]; + type: ExportType; +} + +/** + * Mutation for fetching export data for download. + * Returns a table containing the CSV contents. + */ +export const useDataExportMutation = (): UseMutationResult => { + const mutationResult = useMutation( + async (body: DataExportMutationRequest) => { + const endpoint = endpointWithQueryParams( + normalizeEndpoint("/export"), + new URLSearchParams({ + courses: body.courses.join(","), + fields: body.fields.join(","), + type: body.type + }) + ); + + // open csv file endpoint + window.open(endpoint, "_blank"); + return; + } + ); + + handleError(mutationResult); + return mutationResult; +}; diff --git a/csm_web/frontend/static/frontend/img/file-export.svg b/csm_web/frontend/static/frontend/img/file-export.svg new file mode 100644 index 00000000..43ffa9fa --- /dev/null +++ b/csm_web/frontend/static/frontend/img/file-export.svg @@ -0,0 +1 @@ + diff --git a/csm_web/frontend/static/frontend/img/refresh.svg b/csm_web/frontend/static/frontend/img/refresh.svg new file mode 100644 index 00000000..22c0bf3d --- /dev/null +++ b/csm_web/frontend/static/frontend/img/refresh.svg @@ -0,0 +1 @@ + diff --git a/csm_web/scheduler/urls.py b/csm_web/scheduler/urls.py index b997222f..8c60cd33 100644 --- a/csm_web/scheduler/urls.py +++ b/csm_web/scheduler/urls.py @@ -23,4 +23,5 @@ path("matcher//mentors/", views.matcher.mentors), path("matcher//configure/", views.matcher.configure), path("matcher//create/", views.matcher.create), + path("export/", views.export_data), ] diff --git a/csm_web/scheduler/views/__init__.py b/csm_web/scheduler/views/__init__.py index 8002052a..55ed65f3 100644 --- a/csm_web/scheduler/views/__init__.py +++ b/csm_web/scheduler/views/__init__.py @@ -1,5 +1,6 @@ -from .course import CourseViewSet from . import matcher +from .course import CourseViewSet +from .export import export_data from .profile import ProfileViewSet from .resource import ResourceViewSet from .section import SectionViewSet diff --git a/csm_web/scheduler/views/export.py b/csm_web/scheduler/views/export.py new file mode 100644 index 00000000..ea055df2 --- /dev/null +++ b/csm_web/scheduler/views/export.py @@ -0,0 +1,657 @@ +import csv +import datetime +import io +from typing import Generator, Iterable, List, Optional, Tuple + +from django.contrib.postgres.aggregates import ArrayAgg, JSONBAgg +from django.core.exceptions import BadRequest +from django.db.models import CharField, Count, Q, Value +from django.db.models.functions import Concat +from django.http.response import StreamingHttpResponse +from rest_framework.decorators import api_view +from scheduler.models import Attendance, Course, Section, Student + + +@api_view(["GET"]) +def export_data(request): + """ + Endpoint: /api/export + + GET: Returns a CSV file of exported data. + Query parameters: + preview: int or None + if int > 0, then returns only that many entries from the database + courses: int[] + comma-separated list of course ids + fields: str[] + comma-separated list of fields + type: str + type of data to export + """ + + export_type = request.query_params.get("type", None) + courses_str = request.query_params.get("courses", None) + fields_str = request.query_params.get("fields", "") + preview = request.query_params.get("preview", None) + + if courses_str is None or export_type is None: + raise BadRequest( + "Must include `courses` and `type` fields in the query parameters" + ) + + # convert courses query param into a list of ints + try: + courses = [int(course_id) for course_id in courses_str.split(",")] + except ValueError as exc: + raise BadRequest( + "`courses` query parameter must be a comma-separated list of integers" + ) from exc + fields = fields_str.split(",") + + # check course ids against the user's coordinator courses + coordinator_courses = set( + request.user.coordinator_set.values_list("course__id", flat=True) + ) + courses_set_diff = set(courses).difference(coordinator_courses) + if len(courses_set_diff) > 0: + raise PermissionError( + "You must be a coordinator for all of the courses in the request" + ) + + # convert preview query param into an int + if preview is not None: + try: + preview = int(preview) + except ValueError as exc: + raise BadRequest( + "`preview` query parameter must be an integer or excluded" + ) from exc + + if preview <= 0: + preview = None + + # create generator for the CSV file + csv_generator, filename = prepare_csv(export_type, courses, fields, preview=preview) + + # stream the response; this allows for more efficient data return + response = StreamingHttpResponse( + csv_generator, + content_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + return response + + +def get_section_times_dict(courses: List[int], section_ids: Iterable[int]): + """ + Query the database for section times data, restricting to the given iterable of section ids. + + Normally, all data fields are fetched at the same time in a single query, but + a second aggregate query on a different related field + causes two OUTER JOIN operations in the SQL; this means that + the table size increases multiplicatively, and creates many duplicate items. + An alternative considered is a subquery in the SELECT statement, + but this causes an extra subquery for every returned row in the database. + A last alternative is a subquery to fetch the aggregate data as an INNER JOIN, + but Django does not make it easy to take control of the INNER JOIN calls. + As such, we make a second query to individually fetch the aggregate section time data, + and combine the results in Python. + """ + section_time_queryset = Section.objects.filter( + # filter for courses + mentor__course__id__in=courses + ).annotate( + _num_spacetimes=Count("spacetimes"), + _location=ArrayAgg("spacetimes__location"), + _start_time=ArrayAgg("spacetimes__start_time"), + _duration=ArrayAgg("spacetimes__duration"), + # weird behavior where ArrayAgg with custom DayOfWeekField + # doesn't work (array of enums seems to be parsed by character + # and returned as a single string) + _day=JSONBAgg("spacetimes__day_of_week"), + ) + + # filter for only students in the earlier queryset + # (some may be omitted by the preview or if the section is empty) + section_time_queryset = section_time_queryset.filter(id__in=section_ids) + + section_time_values = section_time_queryset.values( + "id", + "_location", + "_start_time", + "_duration", + "_day", + "_num_spacetimes", + ) + # format values in a dictionary for efficient lookup + return {d["id"]: d for d in section_time_values} + + +def format_section_times(section_info: dict) -> list[str]: + """ + Format a dictionary of section time info. + + Returns a list of formatted section spacetimes. + """ + # format the section times + locations = section_info["_location"] + start_times: List[datetime.time] = section_info["_start_time"] + durations: List[datetime.timedelta] = section_info["_duration"] + days = section_info["_day"] + + time_list = [] + for loc, day, start, duration in zip(locations, days, start_times, durations): + start_formatted = start.strftime("%I:%M %p") + end_datetime = ( + datetime.datetime.combine(datetime.datetime.today(), start) + duration + ) + end_formatted = end_datetime.time().strftime("%I:%M %p") + time_list.append(f"{loc}, {day} {start_formatted}-{end_formatted}") + + return time_list + + +def prepare_csv( + export_type: str, + courses: List[int], + fields: List[str], + preview: Optional[int] = None, +) -> Tuple[Generator, str]: + """ + Delegate CSV preparation to various other methods. + """ + + if export_type == "ATTENDANCE_DATA": + generator = prepare_attendance_data(courses, fields, preview=preview) + filename = "attendance_data.csv" + elif export_type == "COURSE_DATA": + generator = prepare_course_data(courses, fields, preview=preview) + filename = "course_data.csv" + elif export_type == "SECTION_DATA": + generator = prepare_section_data(courses, fields, preview=preview) + filename = "section_data.csv" + elif export_type == "STUDENT_DATA": + generator = prepare_student_data(courses, fields, preview=preview) + filename = "student_data.csv" + else: + raise BadRequest("Invalid export type") + + return generator, filename + + +def create_csv_dict_writer(fieldnames, **kwargs): + """ + Create a CSV DictWriter, wrapped around an in-memory buffer. + + All arguments are passed into the DictWriter constructor. + """ + buffer = io.StringIO() + writer = csv.DictWriter(f=buffer, fieldnames=fieldnames, **kwargs) + + def get_data(): + """ + Fetch the current data from the buffer, + and clear it for the next usage. + """ + buffer.seek(0) + data = buffer.read() + buffer.seek(0) + buffer.truncate() + + return data + + return writer, get_data + + +def prepare_attendance_data( + courses: List[int], fields: List[str], preview: Optional[int] = None +): + """ + Prepare attendance data. + Returns a generator for each row of the CSV file. + + Fields: + Required: + - student_email + - student_name + Optional: + - course_name + - active + - section_id + - mentor_email + - mentor_name + - num_present + - num_excused + - num_unexcused + """ + student_queryset = Student.objects.filter(course__id__in=courses).annotate( + full_name=Concat( + "user__first_name", + Value(" "), + "user__last_name", + output_field=CharField(), + ), + attendance_ids=ArrayAgg("attendance"), + ) + + export_fields = ["user__email", "full_name"] + export_headers = ["Email", "Name"] + + if "course_name" in fields: + export_fields.append("course__name") + export_headers.append("Course") + if "active" in fields: + export_fields.append("active") + export_headers.append("Active") + if "section_id" in fields: + export_fields.append("section__id") + export_headers.append("Section ID") + if "mentor_email" in fields: + export_fields.append("section__mentor__user__email") + export_headers.append("Mentor email") + if "mentor_name" in fields: + student_queryset = student_queryset.annotate( + mentor_name=Concat( + "section__mentor__user__first_name", + Value(" "), + "section__mentor__user__last_name", + output_field=CharField(), + ) + ) + export_fields.append("mentor_name") + export_headers.append("Mentor name") + if "num_present" in fields: + student_queryset = student_queryset.annotate( + num_present=Count("attendance", filter=Q(attendance__presence="PR")) + ) + export_fields.append("num_present") + export_headers.append("Present count") + if "num_unexcused" in fields: + student_queryset = student_queryset.annotate( + num_unexcused=Count("attendance", filter=Q(attendance__presence="UN")) + ) + export_fields.append("num_unexcused") + export_headers.append("Unexcused count") + if "num_excused" in fields: + student_queryset = student_queryset.annotate( + num_excused=Count("attendance", filter=Q(attendance__presence="EX")) + ) + export_fields.append("num_excused") + export_headers.append("Excused count") + + if preview is not None and preview > 0: + # limit queryset + student_queryset = student_queryset[:preview] + + student_values = student_queryset.values(*export_fields, "attendance_ids") + + attendance_ids = set() + for student in student_values: + attendance_ids.update(student["attendance_ids"]) + + attendance_queryset = Attendance.objects.filter( + id__in=attendance_ids + ).select_related("sectionOccurrence") + + attendance_values = attendance_queryset.values( + "id", "presence", "sectionOccurrence__date" + ) + + # preprocess to get all possible columns + attendance_dict = {} + date_set = set() + for attendance in attendance_values: + attendance_dict[attendance["id"]] = attendance + date_set.add(attendance["sectionOccurrence__date"]) + + sorted_dates = sorted(date_set) + + sorted_iso_dates = [date.isoformat() for date in sorted_dates] + header_row = export_fields + sorted_iso_dates + header_desc = export_headers + sorted_iso_dates + csv_writer, get_formatted_row = create_csv_dict_writer(header_row) + + header_dict = dict(zip(header_row, header_desc)) + csv_writer.writerow(header_dict) + yield get_formatted_row() + + for student in student_values: + # initialize row + row = {k: v for k, v in student.items() if k in export_fields} + row.update({iso_date: "" for iso_date in sorted_iso_dates}) + + for attendance_id in student["attendance_ids"]: + if attendance_id is None: + continue + + attendance = attendance_dict[attendance_id] + att_date = attendance["sectionOccurrence__date"] + att_presence = attendance["presence"] + + row[att_date.isoformat()] = att_presence + + csv_writer.writerow(row) + yield get_formatted_row() + + +def prepare_course_data( + courses: List[int], fields: List[str], preview: Optional[int] = None +): + """ + Prepare course data. + Returns a generator for each row of the CSV file. + + Fields: + Required: + - course_name + Optional: + - course_id + - description + - num_sections + - num_students + - num_mentors + """ + + course_queryset = Course.objects.filter(id__in=courses) + + export_fields = ["name"] + export_headers = ["Name"] + if "course_id" in fields: + export_fields.append("id") + export_headers.append("Course ID") + if "description" in fields: + export_fields.append("title") + export_headers.append("Course title") + if "num_sections" in fields: + course_queryset = course_queryset.annotate( + num_sections=Count("mentor__section") + ) + export_fields.append("num_sections") + export_headers.append("Number of sections") + if "num_students" in fields: + course_queryset = course_queryset.annotate( + num_students=Count("mentor__section__students") + ) + export_fields.append("num_students") + export_headers.append("Number of students") + if "num_mentors" in fields: + course_queryset = course_queryset.annotate(num_mentors=Count("mentor")) + export_fields.append("num_mentors") + export_headers.append("Number of mentors") + + if preview is not None and preview > 0: + # limit queryset + course_queryset = course_queryset[:preview] + + values = course_queryset.values(*export_fields) + + csv_writer, get_formatted_row = create_csv_dict_writer(export_fields) + + # write the header row + csv_writer.writerow(dict(zip(export_fields, export_headers))) + yield get_formatted_row() + + # write the remaining rows + for row in values: + csv_writer.writerow(row) + yield get_formatted_row() + + +def prepare_section_data( + courses: List[int], fields: List[str], preview: Optional[int] = None +): + """ + Prepare section data. + Returns a generator for each row of the CSV file. + + Fields: + Required: + - mentor_email + - mentor_name + Optional: + - course_name + - section_id + - section_times + - section_description + - num_students + - capacity + """ + section_queryset = Section.objects.filter(mentor__course__id__in=courses).annotate( + mentor_name=Concat( + "mentor__user__first_name", + Value(" "), + "mentor__user__last_name", + output_field=CharField(), + ) + ) + + export_fields = ["mentor__user__email", "mentor_name"] + export_headers = ["Mentor email", "Mentor name"] + + if "course_name" in fields: + export_fields.append("mentor__course__name") + export_headers.append("Course") + if "section_id" in fields: + export_fields.append("id") + export_headers.append("Section ID") + if "section_description" in fields: + export_fields.append("description") + export_headers.append("Description") + if "num_students" in fields: + section_queryset = section_queryset.annotate(num_students=Count("students")) + export_fields.append("num_students") + export_headers.append("Student count") + if "capacity" in fields: + export_fields.append("capacity") + export_headers.append("Capacity") + + if preview is not None and preview > 0: + # limit queryset + section_queryset = section_queryset[:preview] + + # query database for values; always fetch id + values = section_queryset.values("id", *export_fields) + + section_time_dict = {} + max_spacetime_count = 0 + if "section_times" in fields: + used_ids = set(d["id"] for d in values) + section_time_dict = get_section_times_dict(courses, used_ids) + + # get the maximum number of section spacetimes + if len(section_time_dict) > 0: + max_spacetime_count = max( + d["_num_spacetimes"] for d in section_time_dict.values() + ) + + # these appends are only for the csv writer + if max_spacetime_count > 1: + for spacetime_idx in range(1, max_spacetime_count + 1): + export_fields.append(f"section_times_{spacetime_idx}") + export_headers.append(f"Section times ({spacetime_idx})") + else: + # if there is zero or one spacetime, the header doesn't need to differentiate + # between indices; we still keep the index in the raw field, + # to simplify the code in writing to the csv + export_fields.append("section_times_1") + export_headers.append("Section times") + + csv_writer, get_formatted_row = create_csv_dict_writer(export_fields) + + # write the header row + csv_writer.writerow(dict(zip(export_fields, export_headers))) + yield get_formatted_row() + + # write the remaining rows + for row in values: + # filter out unwanted fields (id, etc.) + final_row = {k: v for k, v in row.items() if k in export_fields} + if "section_times" in fields: + # fetch section info from auxiliary query + section_info = section_time_dict[row["id"]] + formatted_times = format_section_times(section_info) + + # write formatted spacetimes in separate columns + for spacetime_idx in range(max_spacetime_count): + cur_formatted = "" # default to empty string to pad extras + if spacetime_idx < len(formatted_times): + cur_formatted = formatted_times[spacetime_idx] + final_row[f"section_times_{spacetime_idx + 1}"] = cur_formatted + + csv_writer.writerow(final_row) + yield get_formatted_row() + + +def prepare_student_data( + courses: List[int], fields: List[str], preview: Optional[int] = None +): + """ + Prepare student data. + Returns a generator for each row of the CSV file. + + Fields: + Required: + - student_email + - student_name + Optional: + - course_name + - active + - mentor_email + - mentor_name + - section_id + - section_times + - num_present + - num_excused + - num_unexcused + """ + # include the full name in the student queryset by default + # (email is already included as user__email) + student_queryset = Student.objects.filter(course__id__in=courses).annotate( + full_name=Concat( + "user__first_name", Value(" "), "user__last_name", output_field=CharField() + ) + ) + + # fields to fetch from the database + export_qs_fields = ["user__email", "full_name", "section__id"] + # fields to use for the CSV file; must correspond exactly to export_headers + export_fields = ["user__email", "full_name"] + # headers to use for the CSV file; must correspond exactly to export_fields + export_headers = ["Email", "Name"] + + if "course_name" in fields: + export_fields.append("course__name") + export_qs_fields.append("course__name") + export_headers.append("Course") + if "active" in fields: + export_fields.append("active") + export_qs_fields.append("active") + export_headers.append("Active") + if "mentor_email" in fields: + export_fields.append("section__mentor__user__email") + export_qs_fields.append("section__mentor__user__email") + export_headers.append("Mentor email") + if "mentor_name" in fields: + student_queryset = student_queryset.annotate( + mentor_name=Concat( + "section__mentor__user__first_name", + Value(" "), + "section__mentor__user__last_name", + output_field=CharField(), + ) + ) + export_fields.append("mentor_name") + export_qs_fields.append("mentor_name") + export_headers.append("Mentor name") + if "section_id" in fields: + export_fields.append("section__id") + export_headers.append("Section ID") + + if "num_present" in fields: + student_queryset = student_queryset.annotate( + num_present=Count("attendance", filter=Q(attendance__presence="PR")) + ) + export_fields.append("num_present") + export_qs_fields.append("num_present") + export_headers.append("Present count") + if "num_unexcused" in fields: + student_queryset = student_queryset.annotate( + num_unexcused=Count("attendance", filter=Q(attendance__presence="UN")) + ) + export_fields.append("num_unexcused") + export_qs_fields.append("num_unexcused") + export_headers.append("Unexcused count") + if "num_excused" in fields: + student_queryset = student_queryset.annotate( + num_excused=Count("attendance", filter=Q(attendance__presence="EX")) + ) + export_fields.append("num_excused") + export_qs_fields.append("num_excused") + export_headers.append("Excused count") + + if preview is not None and preview > 0: + # limit queryset + student_queryset = student_queryset[:preview] + + # query database for values + values = student_queryset.values(*export_qs_fields) + + # default empty dict (not used if section_times is not specified) + section_time_dict = {} + max_spacetime_count = 0 + if "section_times" in fields: + # A second aggregate query on a different related field + # causes two OUTER JOIN operations in the SQL; this means that + # the table size increases multiplicatively, and creates many duplicate items. + # An alternative considered is a subquery in the SELECT statement, + # but this causes an extra subquery for every returned row in the database. + # A last alternative is a subquery to fetch the aggregate data as an INNER JOIN, + # but Django does not make it easy to take control of the INNER JOIN calls. + # As such, we make a second query to individually fetch the aggregate section time data, + # and combine the results in Python. + + used_ids = set(d["section__id"] for d in values) + section_time_dict = get_section_times_dict(courses, used_ids) + + # get the maximum number of section spacetimes + if len(section_time_dict) > 0: + max_spacetime_count = max( + d["_num_spacetimes"] for d in section_time_dict.values() + ) + + # these appends are only for the csv writer + if max_spacetime_count > 1: + for spacetime_idx in range(max_spacetime_count): + export_fields.append(f"section_times_{spacetime_idx + 1}") + export_headers.append(f"Section times ({spacetime_idx + 1})") + else: + # if there is zero or one spacetime, the header doesn't need to differentiate + # between indices; we still keep the index in the raw field, + # to simplify the code in writing to the csv + export_fields.append("section_times_1") + export_headers.append("Section times") + + csv_writer, get_formatted_row = create_csv_dict_writer(export_fields) + + # write the header row + csv_writer.writerow(dict(zip(export_fields, export_headers))) + yield get_formatted_row() + + # write the remaining rows + for row in values: + # filter out unwanted fields (id, etc.) + final_row = {k: v for k, v in row.items() if k in export_fields} + if "section_times" in fields: + # fetch section info from auxiliary query + section_info = section_time_dict[row["section__id"]] + formatted_times = format_section_times(section_info) + # write formatted spacetimes in separate columns + for spacetime_idx in range(max_spacetime_count): + cur_formatted = "" # default to empty string to pad extras + if spacetime_idx < len(formatted_times): + cur_formatted = formatted_times[spacetime_idx] + final_row[f"section_times_{spacetime_idx + 1}"] = cur_formatted + + csv_writer.writerow(final_row) + yield get_formatted_row() diff --git a/package-lock.json b/package-lock.json index 6ce9e501..974df6c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,6 +48,7 @@ "css-loader": "^6.8.1", "css-minimizer-webpack-plugin": "^5.0.1", "csso-cli": "^3.0.0", + "csv-parse": "^5.5.3", "cypress": "^13.6.1", "cypress-pipe": "^2.0.0", "eslint": "^8.50.0", @@ -6464,6 +6465,12 @@ "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.8.tgz", "integrity": "sha512-jXKhWqXPmlUeoQnF/EhTtTl4C9SnrxSH/jZUih3jmO6lBKr99rP3/+FmrMj4EFpOXzMtXHAZkd3x0E6h6Fgflw==" }, + "node_modules/csv-parse": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz", + "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A==", + "dev": true + }, "node_modules/cypress": { "version": "13.6.1", "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.6.1.tgz", @@ -21247,6 +21254,12 @@ "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.8.tgz", "integrity": "sha512-jXKhWqXPmlUeoQnF/EhTtTl4C9SnrxSH/jZUih3jmO6lBKr99rP3/+FmrMj4EFpOXzMtXHAZkd3x0E6h6Fgflw==" }, + "csv-parse": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz", + "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A==", + "dev": true + }, "cypress": { "version": "13.6.1", "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.6.1.tgz", diff --git a/package.json b/package.json index 0312eab6..ab9d8732 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "css-loader": "^6.8.1", "css-minimizer-webpack-plugin": "^5.0.1", "csso-cli": "^3.0.0", + "csv-parse": "^5.5.3", "cypress": "^13.6.1", "cypress-pipe": "^2.0.0", "eslint": "^8.50.0",