From a123fd3426813e3f29e05c8254327da38d0b3432 Mon Sep 17 00:00:00 2001 From: Nathanp9000 Date: Wed, 8 Apr 2026 17:07:45 -0500 Subject: [PATCH 1/5] Scraping program discovery page. Next will go to a returned program and analyze structure to make a good schema. --- scrapers/schema.go | 126 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 scrapers/schema.go diff --git a/scrapers/schema.go b/scrapers/schema.go new file mode 100644 index 00000000..4839b068 --- /dev/null +++ b/scrapers/schema.go @@ -0,0 +1,126 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/url" + "strconv" + "strings" + "time" + + "github.com/chromedp/cdproto/network" + "github.com/chromedp/chromedp" +) + +const ( + discoveryURL = "https://utdallas-ea.terradotta.com/_portal/program-discovery" + discoveryEndpoint = "/program-api/program/discovery-search/" +) + +type discoverySearchResponse struct { + ReturnedPrograms []int `json:"returnedPrograms"` + ResultCount int `json:"resultCount"` + MaxScore float64 `json:"maxScore"` + Start int `json:"start"` + Limit int `json:"limit"` + TotalCount int `json:"totalCount"` + SearchID string `json:"searchid"` +} + +func main() { + ctx, cancel := chromedp.NewContext(context.Background()) + defer cancel() + + ctx, cancel = context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + var brochureLinks []string + var discoveryBody []byte + var discoveryRequestID network.RequestID + + chromedp.ListenTarget(ctx, func(ev interface{}) { + switch ev := ev.(type) { + case *network.EventResponseReceived: + if strings.Contains(ev.Response.URL, discoveryEndpoint) { + discoveryRequestID = ev.RequestID + } + } + }) + + err := chromedp.Run(ctx, + network.Enable(), + chromedp.Navigate(discoveryURL), + chromedp.WaitVisible("body", chromedp.ByQuery), + // Insert filter/search actions here before waiting for results. + chromedp.Sleep(8*time.Second), + chromedp.ActionFunc(func(ctx context.Context) error { + if discoveryRequestID == "" { + return nil + } + + var err error + discoveryBody, err = network.GetResponseBody(discoveryRequestID).Do(ctx) + return err + }), + chromedp.Evaluate(` + Array.from(document.querySelectorAll("a")) + .map(a => a.href) + .filter(href => href.includes("tds-program-brochure")) + `, &brochureLinks), + ) + if err != nil { + log.Fatal(err) + } + + if len(discoveryBody) > 0 { + var discovery discoverySearchResponse + if err := json.Unmarshal(discoveryBody, &discovery); err != nil { + log.Fatalf("parse discovery response: %v", err) + } + fmt.Printf("searchID=%s total=%d start=%d limit=%d returned=%d\n", + discovery.SearchID, + discovery.TotalCount, + discovery.Start, + discovery.Limit, + discovery.ResultCount, + ) + fmt.Printf("programIDs=%v\n", discovery.ReturnedPrograms) + return + } + + programIDs := extractProgramIDs(brochureLinks) + if len(programIDs) == 0 { + log.Fatal("did not capture discovery-search response or any brochure links") + } + + fmt.Printf("discovery response was not captured; using DOM fallback\n") + fmt.Printf("programIDs=%v\n", programIDs) +} + +func extractProgramIDs(links []string) []int { + ids := make([]int, 0, len(links)) + seen := make(map[int]struct{}, len(links)) + + for _, rawLink := range links { + parsed, err := url.Parse(rawLink) + if err != nil { + continue + } + + programID, err := strconv.Atoi(parsed.Query().Get("programid")) + if err != nil { + continue + } + + if _, ok := seen[programID]; ok { + continue + } + + seen[programID] = struct{}{} + ids = append(ids, programID) + } + + return ids +} From a6d2bb8f063351cda443b02c0e2e91cc7cc70d03 Mon Sep 17 00:00:00 2001 From: Nathanp9000 Date: Thu, 9 Apr 2026 06:26:34 -0500 Subject: [PATCH 2/5] Fetch Brochure function works --- scrapers/schema.go | 74 +++++++++++++++++++++++++++++++++++++++++----- utils/methods.go | 57 +++++++++++++++++++++++++---------- 2 files changed, 108 insertions(+), 23 deletions(-) diff --git a/scrapers/schema.go b/scrapers/schema.go index 4839b068..0a3a60ad 100644 --- a/scrapers/schema.go +++ b/scrapers/schema.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "fmt" + "io" "log" + "net/http" "net/url" "strconv" "strings" @@ -29,6 +31,11 @@ type discoverySearchResponse struct { SearchID string `json:"searchid"` } +type brochure struct { + ProgramID int + Content string +} + func main() { ctx, cancel := chromedp.NewContext(context.Background()) defer cancel() @@ -39,6 +46,7 @@ func main() { var brochureLinks []string var discoveryBody []byte var discoveryRequestID network.RequestID + var programIDs []int chromedp.ListenTarget(ctx, func(ev interface{}) { switch ev := ev.(type) { @@ -86,17 +94,28 @@ func main() { discovery.Limit, discovery.ResultCount, ) - fmt.Printf("programIDs=%v\n", discovery.ReturnedPrograms) - return - } + programIDs = discovery.ReturnedPrograms + fmt.Printf("programIDs=%v\n", programIDs) + } else { + programIDs = extractProgramIDs(brochureLinks) + if len(programIDs) == 0 { + log.Fatal("did not capture discovery-search response or any brochure links") + } - programIDs := extractProgramIDs(brochureLinks) - if len(programIDs) == 0 { - log.Fatal("did not capture discovery-search response or any brochure links") + fmt.Printf("discovery response was not captured; using DOM fallback\n") + fmt.Printf("programIDs=%v\n", programIDs) } - fmt.Printf("discovery response was not captured; using DOM fallback\n") - fmt.Printf("programIDs=%v\n", programIDs) + for _, programID := range programIDs { + brochure, err := fetchBrochure(programID) + if err != nil { + log.Printf("fetch brochure %d: %v", programID, err) + continue + } + + fmt.Printf("\nBROCHURE %d\n", brochure.ProgramID) + fmt.Printf("content=%s\n", brochure.Content) + } } func extractProgramIDs(links []string) []int { @@ -124,3 +143,42 @@ func extractProgramIDs(links []string) []int { return ids } + +func fetchBrochure(programID int) (brochure, error) { + brochureURL := fmt.Sprintf( + "https://utdallas-ea.terradotta.com/_portal/tds-program-brochure?programid=%d", + programID, + ) + + client := &http.Client{ + Timeout: 30 * time.Second, + } + + req, err := http.NewRequest(http.MethodGet, brochureURL, nil) + if err != nil { + return brochure{}, err + } + + req.Header.Set("User-Agent", "Mozilla/5.0") + + resp, err := client.Do(req) + if err != nil { + return brochure{}, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return brochure{}, fmt.Errorf("status %s: %s", resp.Status, strings.TrimSpace(string(body))) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return brochure{}, err + } + + return brochure{ + ProgramID: programID, + Content: string(body), + }, nil +} diff --git a/utils/methods.go b/utils/methods.go index addb0007..181120b0 100644 --- a/utils/methods.go +++ b/utils/methods.go @@ -33,24 +33,51 @@ func GetEnv(name string) (string, error) { return value, nil } -// InitChromeDp configures and returns a chromedp context with optional headless settings. -func InitChromeDp() (chromedpCtx context.Context, cancelFnc context.CancelFunc) { +func InitChromeDp() (context.Context, context.CancelFunc) { log.Printf("Initializing chromedp...") - if Headless { - opts := append(chromedp.DefaultExecAllocatorOptions[:], - chromedp.Flag("headless", true), - chromedp.Flag("no-sandbox", true), - chromedp.Flag("disable-dev-shm-usage", true), - chromedp.Flag("disable-gpu", true), - ) - allocCtx, _ := chromedp.NewExecAllocator(context.Background(), opts...) - chromedpCtx, cancelFnc = chromedp.NewContext(allocCtx) - } else { - allocCtx, _ := chromedp.NewExecAllocator(context.Background()) - chromedpCtx, cancelFnc = chromedp.NewContext(allocCtx) + + // Detect browser path + chromePath := os.Getenv("CHROME_PATH") + + if chromePath == "" { + candidates := []string{ + "google-chrome", + "chromium", + "chromium-browser", + } + + for _, name := range candidates { + if p, err := exec.LookPath(name); err == nil { + chromePath = p + break + } + } + } + + if chromePath == "" { + log.Fatal("No Chrome/Chromium found. Install one or set CHROME_PATH") + } + + log.Printf("Using Chrome at: %s", chromePath) + + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.ExecPath(chromePath), + chromedp.Flag("headless", false), + chromedp.Flag("no-sandbox", true), + chromedp.Flag("disable-dev-shm-usage", true), + chromedp.Flag("disable-gpu", true), + ) + + allocCtx, cancelAlloc := chromedp.NewExecAllocator(context.Background(), opts...) + ctx, cancelCtx := chromedp.NewContext(allocCtx) + + cancel := func() { + cancelCtx() + cancelAlloc() } + log.Printf("Initialized chromedp!") - return chromedpCtx, cancelFnc + return ctx, cancel } // RefreshToken logs into CourseBook and returns headers containing a fresh session token. From 20deabc21a3031f793f24add64ec770da2bcbd3f Mon Sep 17 00:00:00 2001 From: Nathanp9000 Date: Tue, 20 Oct 2026 21:25:44 -0500 Subject: [PATCH 3/5] Switch to nebula-api rest/schema module path nebula-api moved api/ to rest/ and the module is now github.com/UTDNebula/nebula-api. Co-Authored-By: Claude Opus 5.5 --- go.mod | 76 ++++++------ go.sum | 174 ++++++++++++++------------- parser/academicCalendarsParser.go | 2 +- parser/astraParser.go | 2 +- parser/budgetsParser.go | 2 +- parser/cometCalendarParser.go | 2 +- parser/cometCalendarParser_test.go | 2 +- parser/courseParser.go | 2 +- parser/courseParser_test.go | 2 +- parser/degreeParser.go | 2 +- parser/discountsParser.go | 2 +- parser/discountsParser_test.go | 2 +- parser/mapParser.go | 2 +- parser/mazevoParser.go | 2 +- parser/parser.go | 2 +- parser/parser_test.go | 2 +- parser/professorParser.go | 2 +- parser/profileLoader.go | 2 +- parser/requisiteParser.go | 2 +- parser/requisiteParser_test.go | 2 +- parser/sectionParser.go | 2 +- parser/validator.go | 2 +- parser/validator_test.go | 2 +- scrapers/cometCalendar.go | 2 +- scrapers/profiles.go | 2 +- uploader/academicCalendarUploader.go | 2 +- uploader/budgetsUploader.go | 2 +- uploader/degreesUploader.go | 2 +- uploader/discountsUploader.go | 2 +- uploader/discountsUploader_test.go | 2 +- uploader/eventsUploader.go | 2 +- uploader/mapUploader.go | 2 +- uploader/uploader.go | 2 +- utils/parser.go | 2 +- 34 files changed, 158 insertions(+), 156 deletions(-) diff --git a/go.mod b/go.mod index f53b2c23..3c6e7925 100644 --- a/go.mod +++ b/go.mod @@ -1,45 +1,45 @@ module github.com/UTDNebula/api-tools -go 1.26 +go 1.26.0 require ( github.com/PuerkitoBio/goquery v1.12.0 - github.com/UTDNebula/nebula-api/api v0.0.0-20260525053158-3209b0868dcf + github.com/UTDNebula/nebula-api v1.2.1-0.20261021022412-cc9158c9cf75 github.com/chromedp/cdproto v0.0.0-20260321001828-e3e3800016bc github.com/chromedp/chromedp v0.15.1 github.com/dongri/phonenumber v0.1.12 - github.com/getsentry/sentry-go v0.46.2 + github.com/getsentry/sentry-go v0.48.0 github.com/google/go-cmp v0.7.0 github.com/joho/godotenv v1.5.1 github.com/valyala/fastjson v1.6.10 go.mongodb.org/mongo-driver v1.17.9 - golang.org/x/net v0.55.0 - golang.org/x/text v0.39.0 + golang.org/x/net v0.58.0 + golang.org/x/text v0.41.0 google.golang.org/genai v1.51.0 ) require ( - cel.dev/expr v0.25.1 // indirect + cel.dev/expr v0.25.3 // indirect cloud.google.com/go v0.123.0 // indirect - cloud.google.com/go/auth v0.19.0 // indirect + cloud.google.com/go/auth v0.23.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.6.0 // indirect - cloud.google.com/go/monitoring v1.24.3 // indirect - cloud.google.com/go/storage v1.61.3 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + cloud.google.com/go/iam v1.13.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect + cloud.google.com/go/storage v1.65.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.36.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.60.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.60.0 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chromedp/sysutil v1.1.0 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.39.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect @@ -47,35 +47,35 @@ require ( github.com/golang/snappy v1.0.0 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect - github.com/googleapis/gax-go/v2 v2.20.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect + github.com/googleapis/gax-go/v2 v2.24.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect - github.com/klauspost/compress v1.18.5 // indirect - github.com/montanaflynn/stats v0.9.0 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/montanaflynn/stats v0.12.4 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect - go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect - golang.org/x/crypto v0.51.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + golang.org/x/crypto v0.55.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.45.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/time v0.15.0 // indirect - google.golang.org/api v0.273.0 // indirect - google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/grpc v1.79.3 // indirect - google.golang.org/protobuf v1.36.11 // indirect + google.golang.org/api v0.293.0 // indirect + google.golang.org/genproto v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/go.sum b/go.sum index 3a460816..a1735378 100644 --- a/go.sum +++ b/go.sum @@ -1,37 +1,37 @@ -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.19.0 h1:DGYwtbcsGsT1ywuxsIoWi1u/vlks0moIblQHgSDgQkQ= -cloud.google.com/go/auth v0.19.0/go.mod h1:2Aph7BT2KnaSFOM0JDPyiYgNh6PL9vGMiP8CUIXZ+IY= +cloud.google.com/go/auth v0.23.2 h1:pxSCpfiji41hpzpPdMCftEUCezpgpqmmDdYiAjCKXxo= +cloud.google.com/go/auth v0.23.2/go.mod h1:4DhBRcqvtljQN3dJ57qtqbib5ZGCYE5f2crfiiC2EM0= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.6.0 h1:JiSIcEi38dWBKhB3BtfKCW+dMvCZJEhBA2BsaGJgoxs= -cloud.google.com/go/iam v1.6.0/go.mod h1:ZS6zEy7QHmcNO18mjO2viYv/n+wOUkhJqGNkPPGueGU= -cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= -cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= -cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= -cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= -cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= -cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= -cloud.google.com/go/storage v1.61.3 h1:VS//ZfBuPGDvakfD9xyPW1RGF1Vy3BWUoVZXgW1KMOg= -cloud.google.com/go/storage v1.61.3/go.mod h1:JtqK8BBB7TWv0HVGHubtUdzYYrakOQIsMLffZ2Z/HWk= -cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= -cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= +cloud.google.com/go/iam v1.13.0 h1:ufT3FPT5rFFXu6UtLkNoxaOaV5EuA1dsSkmemCSTo6U= +cloud.google.com/go/iam v1.13.0/go.mod h1:gHXdDEiPDvqd1q1KwBDGQlgZY/BwY760zU2LhOZS5w0= +cloud.google.com/go/logging v1.19.1 h1:7SsLhyTDBDrJw+Ll6Ns3I2mByqHXvJUc3rGjSlwiWgU= +cloud.google.com/go/logging v1.19.1/go.mod h1:2IkQ/d8jVJqV2qW8ZUGUiMjdZG1gkLD2JReGbZ8isqg= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= +cloud.google.com/go/storage v1.65.0 h1:McbFt5j+hTNx+dkFuzq7teakIKcpqGp/cJZRxMyfvAc= +cloud.google.com/go/storage v1.65.0/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.36.0 h1:3SdxXLkgAfiHRWcGTq6fneq9jgoJzneiY0yPQnjoT2E= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.36.0/go.mod h1:1iIdl0k+ppn9wT0wzR9H7HkSvIui/4qgtnKW10cQtds= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.60.0 h1:HldzheTs05E3ybqSitI/wHaof6+XERRudgZLjYbs3eE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.60.0/go.mod h1:evkqaSczW9g2BQm1veCtgNhJ4wCCsRrOsSgNIn9LHQk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.60.0 h1:Fx8NtDCmKH4ML2hUkPz4Dq250903vRDojMjVCDKwQuc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.60.0/go.mod h1:V9g30lTKzfUsEW+gpWssck6u9IhARajmipodImLLcwI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.60.0 h1:Oblia1QXBJlM/wOY9ARRUtsXdDYiMCzk3eCMikqoLbI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.60.0/go.mod h1:SRAbhyZ4R4FagHMM9VtRgSY/lheRoht2fKelZXQUenk= github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= -github.com/UTDNebula/nebula-api/api v0.0.0-20260525053158-3209b0868dcf h1:5IIliVrXFa8zyLrWkvK6Z5gtXVw8nrdsXWJkZqRONAU= -github.com/UTDNebula/nebula-api/api v0.0.0-20260525053158-3209b0868dcf/go.mod h1:i+PQZZ3qPtE4UxXkp3tQ46NWpzB8Of2/VNl0iZ/uv9I= +github.com/UTDNebula/nebula-api v1.2.1-0.20261021022412-cc9158c9cf75 h1:jbIi5HNMXNBXom0wLNnAvZrqzrFMIYtgzSV9NCgiCXg= +github.com/UTDNebula/nebula-api v1.2.1-0.20261021022412-cc9158c9cf75/go.mod h1:/LZlxS9qTapLyJe+evzTRqcfWOrjC9arBxkmCFJirgs= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -50,16 +50,16 @@ github.com/dongri/phonenumber v0.1.12 h1:rR/4VZzxqpocUdyM4dIdfY0TWd8FcW43oiyPaOU github.com/dongri/phonenumber v0.1.12/go.mod h1:cuHFSstIxh6qh/Qs/SCV3Grb/JMYregBLuXELvSYmT4= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= -github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= -github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/envoy v1.39.0 h1:1uwRDYPYG8BIBU9Mj1sUAebNmlM6beu/ZKKweSLDxk8= +github.com/envoyproxy/go-control-plane/envoy v1.39.0/go.mod h1:5e4ylfTZO723MEEFsCpSW4ZEBWR8mwkEyXfwJBTCZ9c= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/getsentry/sentry-go v0.46.2 h1:1jhYwrKGa3sIpo/y5iDNXS5wDoT7I1KNzMHrnK6ojns= -github.com/getsentry/sentry-go v0.46.2/go.mod h1:evVbw2qotNUdYG8KxXbAdjOQWWvWIwKxpjdZZIvcIPw= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/getsentry/sentry-go v0.48.0 h1:FRZNr7Uk1C86ev1bSJmYlUkL9oyivQA6YOcdYfaaMmY= +github.com/getsentry/sentry-go v0.48.0/go.mod h1:E5UkA5wp1qR2+MDydNYlVeUiNN2xEdjYMidkgf0Qoss= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= @@ -67,8 +67,8 @@ github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9 github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433 h1:vymEbVwYFP/L05h5TKQxvkXoKxNvTpjxYKdF1Nlwuao= github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -90,20 +90,20 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8= -github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= -github.com/googleapis/gax-go/v2 v2.20.0 h1:NIKVuLhDlIV74muWlsMM4CcQZqN6JJ20Qcxd9YMuYcs= -github.com/googleapis/gax-go/v2 v2.20.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4= +github.com/googleapis/enterprise-certificate-proxy v0.3.21 h1:OFdQ3tnCX/zaQ0Cedur3D3z7kI6HiLX9g3TiAN4/DFU= +github.com/googleapis/enterprise-certificate-proxy v0.3.21/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= +github.com/googleapis/gax-go/v2 v2.24.0 h1:myMaPYyF9MecEmvQqMqomIwn9t/4KCZN9qnwsS76wlg= +github.com/googleapis/gax-go/v2 v2.24.0/go.mod h1:IaTHBDd7NHxSCiu0vEs8pQZu4dGZrWwuSoxCnk16OFM= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= -github.com/montanaflynn/stats v0.9.0 h1:tsBJ0RXwph9BmAuFoCmqGv6e8xa0MENQ8m0ptKq29mQ= -github.com/montanaflynn/stats v0.9.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/montanaflynn/stats v0.12.4 h1:amtNRsti20yIhcrkfUJGwoYqBR82jKQFE8SNNYVgGn0= +github.com/montanaflynn/stats v0.12.4/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= @@ -114,8 +114,8 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= -github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4= @@ -133,24 +133,26 @@ go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5 go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= -go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.40.0 h1:ZrPRak/kS4xI3AVXy8F7pipuDXmDsrO8Lg+yQjBLjw0= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.40.0/go.mod h1:3y6kQCWztq6hyW8Z9YxQDDm0Je9AJoFar2G0yDcmhRk= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0 h1:9jR0ZPRok9ryaOQ2Wx8rg5F7Aon59mxrqbVI60/vlBk= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0 h1:hqxVTu/GtBF+vJ8d1fzW7fRxZFvgoDjWcxwwCaFDYpU= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0/go.mod h1:z5fVEF4X5v0ESvlJqBrrFlBVoj5EQuefZpzsu7R+x5Q= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -159,8 +161,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -175,8 +177,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -186,8 +188,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -200,8 +202,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -221,8 +223,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -232,21 +234,21 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/api v0.273.0 h1:r/Bcv36Xa/te1ugaN1kdJ5LoA5Wj/cL+a4gj6FiPBjQ= -google.golang.org/api v0.273.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.293.0 h1:p9XIWOf63U4OgYx120ZwVU8+vl4XTPmWfgVPnmOAS9w= +google.golang.org/api v0.293.0/go.mod h1:6n5tjEB1gzwniZTepZ0g5u+wM7Bof5GeULCx/zh8ZE0= google.golang.org/genai v1.51.0 h1:IZGuUqgfx40INv3hLFGCbOSGp0qFqm7LVmDghzNIYqg= google.golang.org/genai v1.51.0/go.mod h1:A3kkl0nyBjyFlNjgxIwKq70julKbIxpSxqKO5gw/gmk= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 h1:41r6JMbpzBMen0R/4TZeeAmGXSJC7DftGINUodzTkPI= -google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto v0.0.0-20260819154853-08b0e4226688 h1:2ficYAs+4TBEqohUu9hr7J2YeNTie+3BVyqERjs5Hm0= +google.golang.org/genproto v0.0.0-20260819154853-08b0e4226688/go.mod h1:icDJeJwWhZtQDn/1WGql+0n01hbizh4G7/T75RoxcHs= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/parser/academicCalendarsParser.go b/parser/academicCalendarsParser.go index a69ffefd..ed5d39cf 100644 --- a/parser/academicCalendarsParser.go +++ b/parser/academicCalendarsParser.go @@ -22,7 +22,7 @@ import ( "time" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "google.golang.org/genai" ) diff --git a/parser/astraParser.go b/parser/astraParser.go index f2e7890e..7d0564f1 100644 --- a/parser/astraParser.go +++ b/parser/astraParser.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" ) // InputData describes the raw Astra export payload containing fields metadata and row values. diff --git a/parser/budgetsParser.go b/parser/budgetsParser.go index cb84bae5..6a55d5a7 100644 --- a/parser/budgetsParser.go +++ b/parser/budgetsParser.go @@ -22,7 +22,7 @@ import ( "time" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "google.golang.org/genai" ) diff --git a/parser/cometCalendarParser.go b/parser/cometCalendarParser.go index 70ca0ec9..7f66adb7 100644 --- a/parser/cometCalendarParser.go +++ b/parser/cometCalendarParser.go @@ -15,7 +15,7 @@ import ( "github.com/UTDNebula/api-tools/scrapers" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "golang.org/x/text/cases" "golang.org/x/text/language" ) diff --git a/parser/cometCalendarParser_test.go b/parser/cometCalendarParser_test.go index 67573097..cd7216ad 100644 --- a/parser/cometCalendarParser_test.go +++ b/parser/cometCalendarParser_test.go @@ -10,7 +10,7 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "github.com/google/go-cmp/cmp" ) diff --git a/parser/courseParser.go b/parser/courseParser.go index 5bacd98c..516246e6 100644 --- a/parser/courseParser.go +++ b/parser/courseParser.go @@ -8,7 +8,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/parser/courseParser_test.go b/parser/courseParser_test.go index a72ede80..3792a6f7 100644 --- a/parser/courseParser_test.go +++ b/parser/courseParser_test.go @@ -6,7 +6,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" ) // TestGetCourse checks course parsing from HTML fixtures. diff --git a/parser/degreeParser.go b/parser/degreeParser.go index 33f895ef..aa35e843 100644 --- a/parser/degreeParser.go +++ b/parser/degreeParser.go @@ -9,7 +9,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" ) // Parses scarped degree HTML and outputs the data in JSON diff --git a/parser/discountsParser.go b/parser/discountsParser.go index b60a8bb2..4a2cf8a2 100644 --- a/parser/discountsParser.go +++ b/parser/discountsParser.go @@ -11,7 +11,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "github.com/dongri/phonenumber" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/parser/discountsParser_test.go b/parser/discountsParser_test.go index 42ec8059..b67f0a29 100644 --- a/parser/discountsParser_test.go +++ b/parser/discountsParser_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/PuerkitoBio/goquery" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" ) diff --git a/parser/mapParser.go b/parser/mapParser.go index 875569cb..5ef5e84f 100644 --- a/parser/mapParser.go +++ b/parser/mapParser.go @@ -9,7 +9,7 @@ import ( "slices" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" ) // BUILDINGS_CATEGORY_IDS lists category identifiers for academic, administrative, and housing buildings on Concept3D. diff --git a/parser/mazevoParser.go b/parser/mazevoParser.go index 06b26465..489d6eeb 100644 --- a/parser/mazevoParser.go +++ b/parser/mazevoParser.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" ) var buildingRenames = map[string]string{ diff --git a/parser/parser.go b/parser/parser.go index 1dd22417..895c942c 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -11,7 +11,7 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" "github.com/PuerkitoBio/goquery" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" ) var ( diff --git a/parser/parser_test.go b/parser/parser_test.go index c9e3da29..0bb21570 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -15,7 +15,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "go.mongodb.org/mongo-driver/bson/primitive" diff --git a/parser/professorParser.go b/parser/professorParser.go index 40eb3e52..004727cc 100644 --- a/parser/professorParser.go +++ b/parser/professorParser.go @@ -6,7 +6,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/parser/profileLoader.go b/parser/profileLoader.go index 0d16aa7c..01b7818b 100644 --- a/parser/profileLoader.go +++ b/parser/profileLoader.go @@ -6,7 +6,7 @@ import ( "log" "os" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" ) func loadProfiles(inDir string) { diff --git a/parser/requisiteParser.go b/parser/requisiteParser.go index 88f6ff93..6083c032 100644 --- a/parser/requisiteParser.go +++ b/parser/requisiteParser.go @@ -10,7 +10,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" ) /* diff --git a/parser/requisiteParser_test.go b/parser/requisiteParser_test.go index f80eb555..aaef9034 100644 --- a/parser/requisiteParser_test.go +++ b/parser/requisiteParser_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/parser/sectionParser.go b/parser/sectionParser.go index a6d3ac0d..19a31875 100644 --- a/parser/sectionParser.go +++ b/parser/sectionParser.go @@ -7,7 +7,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "go.mongodb.org/mongo-driver/bson/primitive" "golang.org/x/net/html/atom" ) diff --git a/parser/validator.go b/parser/validator.go index a961c9d1..e9380655 100644 --- a/parser/validator.go +++ b/parser/validator.go @@ -5,7 +5,7 @@ import ( "slices" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/parser/validator_test.go b/parser/validator_test.go index e6a60137..0440bed3 100644 --- a/parser/validator_test.go +++ b/parser/validator_test.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/scrapers/cometCalendar.go b/scrapers/cometCalendar.go index a461b4ea..88f133c9 100644 --- a/scrapers/cometCalendar.go +++ b/scrapers/cometCalendar.go @@ -15,7 +15,7 @@ import ( "time" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/scrapers/profiles.go b/scrapers/profiles.go index 59ec59d0..c8ea764c 100644 --- a/scrapers/profiles.go +++ b/scrapers/profiles.go @@ -16,7 +16,7 @@ import ( "strings" "github.com/UTDNebula/api-tools/utils" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "github.com/chromedp/cdproto/cdp" "github.com/chromedp/cdproto/runtime" "github.com/chromedp/chromedp" diff --git a/uploader/academicCalendarUploader.go b/uploader/academicCalendarUploader.go index 803938a3..ad7791eb 100644 --- a/uploader/academicCalendarUploader.go +++ b/uploader/academicCalendarUploader.go @@ -11,7 +11,7 @@ import ( "os" "time" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "github.com/joho/godotenv" ) diff --git a/uploader/budgetsUploader.go b/uploader/budgetsUploader.go index f01d7801..d222be45 100644 --- a/uploader/budgetsUploader.go +++ b/uploader/budgetsUploader.go @@ -11,7 +11,7 @@ import ( "os" "time" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "github.com/joho/godotenv" ) diff --git a/uploader/degreesUploader.go b/uploader/degreesUploader.go index 019b8ea5..a19fbc65 100644 --- a/uploader/degreesUploader.go +++ b/uploader/degreesUploader.go @@ -7,7 +7,7 @@ import ( "os" "time" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" ) const DEGREES_FILE string = "degrees.json" diff --git a/uploader/discountsUploader.go b/uploader/discountsUploader.go index 3ceb0197..0c3d073e 100644 --- a/uploader/discountsUploader.go +++ b/uploader/discountsUploader.go @@ -11,7 +11,7 @@ import ( "os" "time" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" ) // Note that this uploader assumes that the collection names match the names of these files, which they should. diff --git a/uploader/discountsUploader_test.go b/uploader/discountsUploader_test.go index bf85fd84..97b0b4c6 100644 --- a/uploader/discountsUploader_test.go +++ b/uploader/discountsUploader_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" ) diff --git a/uploader/eventsUploader.go b/uploader/eventsUploader.go index dbfea1bc..46711118 100644 --- a/uploader/eventsUploader.go +++ b/uploader/eventsUploader.go @@ -11,7 +11,7 @@ import ( "os" "time" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "github.com/joho/godotenv" ) diff --git a/uploader/mapUploader.go b/uploader/mapUploader.go index 6262abe8..234a2ffd 100644 --- a/uploader/mapUploader.go +++ b/uploader/mapUploader.go @@ -11,7 +11,7 @@ import ( "os" "time" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "github.com/joho/godotenv" ) diff --git a/uploader/uploader.go b/uploader/uploader.go index 12bfe5b1..d8a12cc8 100644 --- a/uploader/uploader.go +++ b/uploader/uploader.go @@ -17,7 +17,7 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" "github.com/UTDNebula/api-tools/uploader/pipelines" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "go.mongodb.org/mongo-driver/bson/primitive" ) diff --git a/utils/parser.go b/utils/parser.go index b0ae6f0d..7e1f9e8e 100644 --- a/utils/parser.go +++ b/utils/parser.go @@ -16,7 +16,7 @@ import ( "strings" - "github.com/UTDNebula/nebula-api/api/schema" + "github.com/UTDNebula/nebula-api/rest/schema" "google.golang.org/genai" ) From 2197099a0fe6bc8535992141e66aa0173dc72d09 Mon Sep 17 00:00:00 2001 From: Nathanp9000 Date: Tue, 20 Oct 2026 21:25:44 -0500 Subject: [PATCH 4/5] Scrape and parse study abroad programs Scraper saves raw discovery records and brochures; parser merges them into schema.Program. Adds -studyAbroad for scrape and parse, and CHROME_PATH support. Co-Authored-By: Claude Opus 5.5 --- .env.template | 4 + README.md | 2 + main.go | 6 + parser/studyAbroadParser.go | 129 ++++++++++++++ scrapers/schema.go | 341 +++++++++++++++++++++++++----------- utils/methods.go | 3 +- 6 files changed, 384 insertions(+), 101 deletions(-) create mode 100644 parser/studyAbroadParser.go diff --git a/.env.template b/.env.template index db8b09b3..f9b87e10 100644 --- a/.env.template +++ b/.env.template @@ -1,4 +1,8 @@ # Scrapers +# Optional: path to a Chrome/Chromium binary for ChromeDP. +# Leave blank to auto-detect google-chrome, chromium, or chromium-browser on PATH. +# Set this if auto-detection picks a browser that fails to launch (e.g. snap-installed Chromium). +CHROME_PATH= LOGIN_NETID= LOGIN_PASSWORD= LOGIN_ASTRA_USERNAME= diff --git a/README.md b/README.md index d78eae06..7e954105 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ Run the tool by changing directory using `cd` to the `api-tools` directory and r | `./api-tools -scrape -map` | Scrapes UTD Map data. | | `./api-tools -scrape -mazevo` | Scrapes Mazevo data. | | `./api-tools -scrape -profiles` | Scrapes UTD professor profiles. | +| `./api-tools -scrape -studyAbroad` | Scrapes study abroad program brochures and discovery records to `[outDir]/studyAbroad`.
• Set `PROGRAM_IDS` to a comma-separated list to scrape specific programs instead of all of them. | | `./api-tools -scrape -headless` | Runs ChromeDP in headless mode. | | `./api-tools -o [directory]` | Sets output directory (default: `./data`). | @@ -87,6 +88,7 @@ Run the tool by changing directory using `cd` to the `api-tools` directory and r | `./api-tools -parse -degrees` | Parses degrees from HTML. | | `./api-tools -parse -map` | Parses UTD Map data. | | `./api-tools -parse -mazevo` | Parses Mazevo data. | +| `./api-tools -parse -studyAbroad` | Combines scraped study abroad brochures and discovery records into `studyAbroad.json`. | | `./api-tools -parse -skipv` | Skips post-parse validation (**use with caution**). | diff --git a/main.go b/main.go index 2789241b..78307653 100644 --- a/main.go +++ b/main.go @@ -67,6 +67,8 @@ func main() { degrees := flag.Bool("degrees", false, "Alongside -scrape, -parse, or -upload. Signifies that the degrees should be scraped/parsed/uploaded.") // Flag for budget scraping budgets := flag.Bool("budgets", false, "Alongside -scrape, -parse, or -upload, signifies that the budgets should be scraped/parsed/uploaded.") + // Flag for study abroad scraping + studyAbroad := flag.Bool("studyAbroad", false, "Alongside -scrape or -parse, signifies that study abroad programs should be scraped/parsed.") // Flags for parsing parse := flag.Bool("parse", false, "Puts the tool into parsing mode.") @@ -143,6 +145,8 @@ func main() { scrapers.ScrapeDegrees(*outDir) case *budgets: scrapers.ScrapeBudgets(*outDir) + case *studyAbroad: + scrapers.ScrapeStudyAbroad(*outDir) default: log.Panic("You must specify which type of scraping you would like to perform with one of the scraping flags!") } @@ -164,6 +168,8 @@ func main() { parser.ParseDegrees(*inDir, *outDir) case *budgets: parser.ParseBudgets(*inDir, *outDir, *budgetsDir, *useBackupBudgets) + case *studyAbroad: + parser.ParseStudyAbroad(*inDir, *outDir) default: parser.Parse(*inDir, *outDir, *gradesDir, *skipValidation) } diff --git a/parser/studyAbroadParser.go b/parser/studyAbroadParser.go new file mode 100644 index 00000000..aed258ca --- /dev/null +++ b/parser/studyAbroadParser.go @@ -0,0 +1,129 @@ +/* + This file contains the code for the study abroad program parser. +*/ + +package parser + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "sort" + + "github.com/UTDNebula/api-tools/utils" + "github.com/UTDNebula/nebula-api/rest/schema" +) + +// discoveryRecord holds the program metadata that only the discovery search returns. +type discoveryRecord struct { + ProgramID int `json:"Program_Id"` + ProgramName string `json:"Program_Name"` + Summary string `json:"Summary"` + Terms []string `json:"Terms"` + Locations []discoveryLocation `json:"stLocations"` +} + +type discoveryLocation struct { + Country string `json:"country"` + City string `json:"city"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` +} + +// ParseStudyAbroad combines the scraped program brochures with their discovery search records +// and writes the resulting programs to {outDir}/studyAbroad.json. +func ParseStudyAbroad(inDir string, outDir string) { + studyAbroadDir := filepath.Join(inDir, "studyAbroad") + + records, err := loadDiscoveryRecords(filepath.Join(studyAbroadDir, "discoveryScraped.json")) + if err != nil { + log.Panic(err) + } + + brochurePaths := utils.GetAllFilesWithExtension(filepath.Join(studyAbroadDir, "brochures"), ".json") + if len(brochurePaths) == 0 { + log.Panicf("No brochures found in %s; run -scrape -studyAbroad first", studyAbroadDir) + } + + programs := make([]schema.Program, 0, len(brochurePaths)) + missingRecords := 0 + + for _, path := range brochurePaths { + program, err := parseBrochure(path) + if err != nil { + log.Printf("parse brochure %s: %v", path, err) + continue + } + + record, ok := records[program.ID] + if !ok { + missingRecords++ + } + program.Name = record.ProgramName + program.Summary = record.Summary + program.Terms = record.Terms + program.Locations = make([]schema.ProgramLocation, 0, len(record.Locations)) + for _, location := range record.Locations { + program.Locations = append(program.Locations, schema.ProgramLocation{ + City: location.City, + Country: location.Country, + Latitude: location.Latitude, + Longitude: location.Longitude, + }) + } + + programs = append(programs, program) + } + + sort.Slice(programs, func(i, j int) bool { return programs[i].ID < programs[j].ID }) + + if missingRecords > 0 { + log.Printf("%d programs had no discovery record, so they have no name, terms, or locations", missingRecords) + } + log.Printf("Parsed %d/%d study abroad programs!", len(programs), len(brochurePaths)) + + if err := utils.WriteJSON(fmt.Sprintf("%s/studyAbroad.json", outDir), programs); err != nil { + log.Panic(err) + } +} + +// loadDiscoveryRecords reads the discovery search records keyed by program ID. +// A missing file isn't an error, since scraping with PROGRAM_IDS skips discovery. +func loadDiscoveryRecords(path string) (map[int]discoveryRecord, error) { + records := make(map[int]discoveryRecord) + + content, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + log.Printf("No discovery records at %s", path) + return records, nil + } + if err != nil { + return nil, err + } + + var list []discoveryRecord + if err := json.Unmarshal(content, &list); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + + for _, record := range list { + records[record.ProgramID] = record + } + return records, nil +} + +func parseBrochure(path string) (schema.Program, error) { + content, err := os.ReadFile(path) + if err != nil { + return schema.Program{}, err + } + + var program schema.Program + if err := json.Unmarshal(content, &program); err != nil { + return schema.Program{}, err + } + return program, nil +} diff --git a/scrapers/schema.go b/scrapers/schema.go index 0a3a60ad..fb7f2824 100644 --- a/scrapers/schema.go +++ b/scrapers/schema.go @@ -1,154 +1,303 @@ -package main +/* + This file contains the code for the study abroad program scraper. +*/ + +package scrapers import ( + "bytes" "context" "encoding/json" "fmt" "io" "log" "net/http" - "net/url" + "os" + "path/filepath" "strconv" "strings" + "sync" "time" + "github.com/UTDNebula/api-tools/utils" "github.com/chromedp/cdproto/network" "github.com/chromedp/chromedp" ) const ( - discoveryURL = "https://utdallas-ea.terradotta.com/_portal/program-discovery" - discoveryEndpoint = "/program-api/program/discovery-search/" + discoveryURL = "https://utdallas-ea.terradotta.com/_portal/program-discovery" + discoveryEndpoint = "/program-api/program/discovery-search/" + discoverySearchURL = "https://us-prod-api-v2.terradotta.com/program-api/program/discovery-search/" + brochureURLFormat = "https://utdallas-ea.terradotta.com/models/services/REST/index.cfm?endpoint=/v3/program/%d/brochure" + + // discoveryPageSize is the number of programs requested per discovery-search page. + // The portal itself uses 12; a larger page just means fewer round trips. + discoveryPageSize = 100 ) +// discoverySearchRequest is the JSON body the discovery portal POSTs to discovery-search. +// Only active (publicly visible) programs are requested, matching the portal's default filter. +type discoverySearchRequest struct { + Filter []discoveryFilter `json:"filter"` + Limit int `json:"limit"` + SortBy string `json:"sortBy"` + SortDirection string `json:"sortDirection"` + Start int `json:"start"` +} + +type discoveryFilter struct { + FilterType string `json:"filterType"` + FilterValues []discoveryFilterValue `json:"filterValues"` +} + +type discoveryFilterValue struct { + Value bool `json:"value"` +} + type discoverySearchResponse struct { - ReturnedPrograms []int `json:"returnedPrograms"` - ResultCount int `json:"resultCount"` - MaxScore float64 `json:"maxScore"` - Start int `json:"start"` - Limit int `json:"limit"` - TotalCount int `json:"totalCount"` - SearchID string `json:"searchid"` + ReturnedPrograms []int `json:"returnedPrograms"` + // Records are kept raw so the parser sees everything discovery returns. + // They carry the program name, terms, and locations, which the brochure endpoint does not. + Records []json.RawMessage `json:"records"` + ResultCount int `json:"resultCount"` + MaxScore float64 `json:"maxScore"` + Start int `json:"start"` + Limit int `json:"limit"` + TotalCount int `json:"totalCount"` + SearchID string `json:"searchid"` } -type brochure struct { - ProgramID int - Content string +// ScrapeStudyAbroad saves every active study abroad program brochure, plus the discovery search +// records that describe them, to {outDir}/studyAbroad for the parser to combine. +// Set PROGRAM_IDS to a comma-separated list to scrape specific programs instead of discovering them all. +func ScrapeStudyAbroad(outDir string) { + programIDs, err := programIDsFromEnv(os.Getenv("PROGRAM_IDS")) + if err != nil { + log.Panic(err) + } + + studyAbroadDir := filepath.Join(outDir, "studyAbroad") + brochureDir := filepath.Join(studyAbroadDir, "brochures") + + if len(programIDs) == 0 { + var records []json.RawMessage + programIDs, records, err = discoverPrograms() + if err != nil { + log.Panic(err) + } + + // A full scrape replaces the previous one, so brochures for programs that are no longer active don't linger. + if err := os.RemoveAll(brochureDir); err != nil { + log.Panic(err) + } + if err := os.MkdirAll(studyAbroadDir, 0777); err != nil { + log.Panic(err) + } + if err := utils.WriteJSON(filepath.Join(studyAbroadDir, "discoveryScraped.json"), records); err != nil { + log.Panic(err) + } + } else { + log.Printf("Using PROGRAM_IDS=%v", programIDs) + } + + if err := os.MkdirAll(brochureDir, 0777); err != nil { + log.Panic(err) + } + + log.Printf("Fetching %d program brochures...", len(programIDs)) + + scraped := 0 + for _, programID := range programIDs { + content, err := fetchBrochure(programID) + if err != nil { + log.Printf("fetch brochure %d: %v", programID, err) + continue + } + + if err := os.WriteFile(filepath.Join(brochureDir, fmt.Sprintf("%d.json", programID)), content, 0644); err != nil { + log.Printf("write brochure %d: %v", programID, err) + continue + } + scraped++ + } + + log.Printf("Scraped %d/%d study abroad programs to %s!", scraped, len(programIDs), studyAbroadDir) } -func main() { - ctx, cancel := chromedp.NewContext(context.Background()) - defer cancel() +// discoverPrograms loads the discovery portal to capture an API token, then pages through +// discovery-search to collect every active program ID along with its raw discovery record. +func discoverPrograms() ([]int, []json.RawMessage, error) { + authorization, err := captureDiscoveryToken() + if err != nil { + return nil, nil, err + } + + client := &http.Client{Timeout: 30 * time.Second} + + programIDs := make([]int, 0) + records := make([]json.RawMessage, 0) + seen := make(map[int]struct{}) + + for start := 0; ; start += discoveryPageSize { + page, err := fetchDiscoveryPage(client, authorization, start) + if err != nil { + return nil, nil, err + } + + if start == 0 { + log.Printf("Found %d active study abroad programs (searchID=%s)", page.TotalCount, page.SearchID) + } + + records = append(records, page.Records...) + + for _, programID := range page.ReturnedPrograms { + if _, ok := seen[programID]; ok { + continue + } + seen[programID] = struct{}{} + programIDs = append(programIDs, programID) + } - ctx, cancel = context.WithTimeout(ctx, 30*time.Second) + // Stop once the API returns an empty page or we've collected everything it reported. + if page.ResultCount == 0 || len(programIDs) >= page.TotalCount { + break + } + } + + return programIDs, records, nil +} + +// captureDiscoveryToken loads the discovery portal in a browser and intercepts the +// Authorization header of the discovery-search request the page makes on its own. +func captureDiscoveryToken() (string, error) { + browserCtx, browserCancel := utils.InitChromeDp() + defer browserCancel() + + ctx, cancel := context.WithTimeout(browserCtx, 60*time.Second) defer cancel() - var brochureLinks []string - var discoveryBody []byte - var discoveryRequestID network.RequestID - var programIDs []int + var mutex sync.Mutex + var token string + captured := make(chan struct{}) chromedp.ListenTarget(ctx, func(ev interface{}) { - switch ev := ev.(type) { - case *network.EventResponseReceived: - if strings.Contains(ev.Response.URL, discoveryEndpoint) { - discoveryRequestID = ev.RequestID - } + request, ok := ev.(*network.EventRequestWillBeSent) + if !ok || !strings.Contains(request.Request.URL, discoveryEndpoint) { + return + } + + authorization, ok := request.Request.Headers["Authorization"].(string) + if !ok || authorization == "" { + return + } + + mutex.Lock() + defer mutex.Unlock() + if token == "" { + token = authorization + close(captured) } }) + log.Println("Loading program discovery portal...") err := chromedp.Run(ctx, network.Enable(), chromedp.Navigate(discoveryURL), chromedp.WaitVisible("body", chromedp.ByQuery), - // Insert filter/search actions here before waiting for results. - chromedp.Sleep(8*time.Second), chromedp.ActionFunc(func(ctx context.Context) error { - if discoveryRequestID == "" { + select { + case <-captured: return nil + case <-ctx.Done(): + return fmt.Errorf("timed out waiting for a discovery-search request") } - - var err error - discoveryBody, err = network.GetResponseBody(discoveryRequestID).Do(ctx) - return err }), - chromedp.Evaluate(` - Array.from(document.querySelectorAll("a")) - .map(a => a.href) - .filter(href => href.includes("tds-program-brochure")) - `, &brochureLinks), ) if err != nil { - log.Fatal(err) + return "", err } - if len(discoveryBody) > 0 { - var discovery discoverySearchResponse - if err := json.Unmarshal(discoveryBody, &discovery); err != nil { - log.Fatalf("parse discovery response: %v", err) - } - fmt.Printf("searchID=%s total=%d start=%d limit=%d returned=%d\n", - discovery.SearchID, - discovery.TotalCount, - discovery.Start, - discovery.Limit, - discovery.ResultCount, - ) - programIDs = discovery.ReturnedPrograms - fmt.Printf("programIDs=%v\n", programIDs) - } else { - programIDs = extractProgramIDs(brochureLinks) - if len(programIDs) == 0 { - log.Fatal("did not capture discovery-search response or any brochure links") - } + mutex.Lock() + defer mutex.Unlock() + if token == "" { + return "", fmt.Errorf("did not capture a discovery-search authorization token") + } + + return token, nil +} - fmt.Printf("discovery response was not captured; using DOM fallback\n") - fmt.Printf("programIDs=%v\n", programIDs) +func fetchDiscoveryPage(client *http.Client, authorization string, start int) (discoverySearchResponse, error) { + body, err := json.Marshal(discoverySearchRequest{ + Filter: []discoveryFilter{{ + FilterType: "programDiscoveryIsActive", + FilterValues: []discoveryFilterValue{{Value: true}}, + }}, + Limit: discoveryPageSize, + SortBy: "_score", + SortDirection: "DESC", + Start: start, + }) + if err != nil { + return discoverySearchResponse{}, err } - for _, programID := range programIDs { - brochure, err := fetchBrochure(programID) - if err != nil { - log.Printf("fetch brochure %d: %v", programID, err) - continue - } + req, err := http.NewRequest(http.MethodPost, discoverySearchURL, bytes.NewReader(body)) + if err != nil { + return discoverySearchResponse{}, err + } + + req.Header.Set("Authorization", authorization) + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + req.Header.Set("Origin", "https://utdallas-ea.terradotta.com") + req.Header.Set("Referer", "https://utdallas-ea.terradotta.com/") + req.Header.Set("User-Agent", "Mozilla/5.0") + + resp, err := client.Do(req) + if err != nil { + return discoverySearchResponse{}, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + preview, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return discoverySearchResponse{}, fmt.Errorf("discovery-search start=%d: status %s: %s", start, resp.Status, strings.TrimSpace(string(preview))) + } - fmt.Printf("\nBROCHURE %d\n", brochure.ProgramID) - fmt.Printf("content=%s\n", brochure.Content) + var page discoverySearchResponse + if err := json.NewDecoder(resp.Body).Decode(&page); err != nil { + return discoverySearchResponse{}, fmt.Errorf("parse discovery response: %w", err) } + + return page, nil } -func extractProgramIDs(links []string) []int { - ids := make([]int, 0, len(links)) - seen := make(map[int]struct{}, len(links)) +func programIDsFromEnv(raw string) ([]int, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } - for _, rawLink := range links { - parsed, err := url.Parse(rawLink) - if err != nil { + parts := strings.Split(raw, ",") + ids := make([]int, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { continue } - programID, err := strconv.Atoi(parsed.Query().Get("programid")) + programID, err := strconv.Atoi(part) if err != nil { - continue - } - - if _, ok := seen[programID]; ok { - continue + return nil, fmt.Errorf("parse PROGRAM_IDS value %q: %w", part, err) } - seen[programID] = struct{}{} ids = append(ids, programID) } - return ids + return ids, nil } -func fetchBrochure(programID int) (brochure, error) { - brochureURL := fmt.Sprintf( - "https://utdallas-ea.terradotta.com/_portal/tds-program-brochure?programid=%d", - programID, - ) +func fetchBrochure(programID int) ([]byte, error) { + brochureURL := fmt.Sprintf(brochureURLFormat, programID) client := &http.Client{ Timeout: 30 * time.Second, @@ -156,29 +305,21 @@ func fetchBrochure(programID int) (brochure, error) { req, err := http.NewRequest(http.MethodGet, brochureURL, nil) if err != nil { - return brochure{}, err + return nil, err } req.Header.Set("User-Agent", "Mozilla/5.0") resp, err := client.Do(req) if err != nil { - return brochure{}, err + return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - return brochure{}, fmt.Errorf("status %s: %s", resp.Status, strings.TrimSpace(string(body))) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return brochure{}, err + return nil, fmt.Errorf("status %s: %s", resp.Status, strings.TrimSpace(string(body))) } - return brochure{ - ProgramID: programID, - Content: string(body), - }, nil + return io.ReadAll(resp.Body) } diff --git a/utils/methods.go b/utils/methods.go index 181120b0..2067ea6e 100644 --- a/utils/methods.go +++ b/utils/methods.go @@ -9,6 +9,7 @@ import ( "io/fs" "log" "os" + "os/exec" "path/filepath" "regexp" "time" @@ -62,7 +63,7 @@ func InitChromeDp() (context.Context, context.CancelFunc) { opts := append(chromedp.DefaultExecAllocatorOptions[:], chromedp.ExecPath(chromePath), - chromedp.Flag("headless", false), + chromedp.Flag("headless", Headless), chromedp.Flag("no-sandbox", true), chromedp.Flag("disable-dev-shm-usage", true), chromedp.Flag("disable-gpu", true), From 3dbf4984aa2c2b0320c05f8845146c97b2ddbe8d Mon Sep 17 00:00:00 2001 From: Nathanp9000 Date: Tue, 27 Oct 2026 07:00:42 -0500 Subject: [PATCH 5/5] Parse brochures into updated study abroad schema --- go.mod | 2 +- go.sum | 4 +- parser/studyAbroadParser.go | 125 +++++++++++++++++++++++++++++++++++- 3 files changed, 126 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3c6e7925..ad19223f 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.0 require ( github.com/PuerkitoBio/goquery v1.12.0 - github.com/UTDNebula/nebula-api v1.2.1-0.20261021022412-cc9158c9cf75 + github.com/UTDNebula/nebula-api v1.2.1-0.20261022120845-36f2148b044a github.com/chromedp/cdproto v0.0.0-20260321001828-e3e3800016bc github.com/chromedp/chromedp v0.15.1 github.com/dongri/phonenumber v0.1.12 diff --git a/go.sum b/go.sum index a1735378..34bda97e 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.60.0/go.mod h1:SRAbhyZ4R4FagHMM9VtRgSY/lheRoht2fKelZXQUenk= github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= -github.com/UTDNebula/nebula-api v1.2.1-0.20261021022412-cc9158c9cf75 h1:jbIi5HNMXNBXom0wLNnAvZrqzrFMIYtgzSV9NCgiCXg= -github.com/UTDNebula/nebula-api v1.2.1-0.20261021022412-cc9158c9cf75/go.mod h1:/LZlxS9qTapLyJe+evzTRqcfWOrjC9arBxkmCFJirgs= +github.com/UTDNebula/nebula-api v1.2.1-0.20261022120845-36f2148b044a h1:CtUGAhEAOqOagkIgZEgxTyLTO0ZKJyrRA+/l2ZgEyQ0= +github.com/UTDNebula/nebula-api v1.2.1-0.20261022120845-36f2148b044a/go.mod h1:/LZlxS9qTapLyJe+evzTRqcfWOrjC9arBxkmCFJirgs= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/parser/studyAbroadParser.go b/parser/studyAbroadParser.go index aed258ca..40e1fdd2 100644 --- a/parser/studyAbroadParser.go +++ b/parser/studyAbroadParser.go @@ -115,15 +115,136 @@ func loadDiscoveryRecords(path string) (map[int]discoveryRecord, error) { return records, nil } +// rawBrochure is the part of a Terradotta brochure that the parser keeps. +type rawBrochure struct { + ProgramID int `json:"programId"` + Current struct { + Sections []rawSection `json:"sections"` + } `json:"current"` + NextCost *rawCost `json:"nextAppCycleCostSheet"` +} + +type rawSection struct { + DisplayName string `json:"sectionDisplayName"` + Widgets []rawWidget `json:"sectionWidgets"` +} + +type rawWidget struct { + ContentType string `json:"contentType"` + ContentHTML string `json:"contentHTML"` + // "" for most widgets, an object holding the parameters for "information sheet" widgets + InformationSheet json.RawMessage `json:"contentInformationSheet"` +} + +type rawCost struct { + Term string `json:"term"` + Year int `json:"year"` + Billable []rawCostItem `json:"billableCostSheetItems"` + NonBillable []rawCostItem `json:"nonBillableCostSheetItems"` + Credits []rawCostItem `json:"creditCostSheetItems"` +} + +type rawCostItem struct { + Name string `json:"costSheetItemName"` + Type string `json:"costSheetItemType"` + Costs []struct { + Key string `json:"costKey"` + Value float64 `json:"costValue"` + } `json:"costs"` +} + func parseBrochure(path string) (schema.Program, error) { content, err := os.ReadFile(path) if err != nil { return schema.Program{}, err } - var program schema.Program - if err := json.Unmarshal(content, &program); err != nil { + var brochure rawBrochure + if err := json.Unmarshal(content, &brochure); err != nil { return schema.Program{}, err } + + program := schema.Program{ + ID: brochure.ProgramID, + Sections: make([]schema.ProgramSection, 0, len(brochure.Current.Sections)), + } + for _, rawSection := range brochure.Current.Sections { + section := schema.ProgramSection{ + DisplayName: rawSection.DisplayName, + Widgets: make([]schema.ProgramWidget, 0, len(rawSection.Widgets)), + } + for _, rawWidget := range rawSection.Widgets { + parameters, err := parseInformationSheet(rawWidget.InformationSheet) + if err != nil { + return schema.Program{}, fmt.Errorf("section %q: %w", rawSection.DisplayName, err) + } + section.Widgets = append(section.Widgets, schema.ProgramWidget{ + ContentType: rawWidget.ContentType, + ContentHTML: rawWidget.ContentHTML, + Parameters: parameters, + }) + } + program.Sections = append(program.Sections, section) + } + + if brochure.NextCost != nil { + program.NextCost = &schema.Cost{ + Term: brochure.NextCost.Term, + Year: brochure.NextCost.Year, + Billable: convertCostItems(brochure.NextCost.Billable), + NonBillable: convertCostItems(brochure.NextCost.NonBillable), + Credits: convertCostItems(brochure.NextCost.Credits), + } + } return program, nil } + +// parseInformationSheet returns the parameters of an information sheet, or nil when the widget has none. +func parseInformationSheet(raw json.RawMessage) ([]schema.Parameter, error) { + if len(raw) == 0 || raw[0] != '{' { + return nil, nil + } + var sheet struct { + Parameters []struct { + Name string `json:"parameterName"` + AssignedValues []any `json:"assignedValues"` // strings, except YESNO parameters which hold bools + } `json:"parameters"` + } + if err := json.Unmarshal(raw, &sheet); err != nil { + return nil, fmt.Errorf("parse information sheet: %w", err) + } + + parameters := make([]schema.Parameter, 0, len(sheet.Parameters)) + for _, rawParameter := range sheet.Parameters { + values := make([]string, 0, len(rawParameter.AssignedValues)) + for _, value := range rawParameter.AssignedValues { + switch value := value.(type) { + case string: + values = append(values, value) + case bool: + if value { + values = append(values, "Yes") + } else { + values = append(values, "No") + } + default: + return nil, fmt.Errorf("parameter %q: unexpected value %v", rawParameter.Name, value) + } + } + parameters = append(parameters, schema.Parameter{Name: rawParameter.Name, AssignedValues: values}) + } + return parameters, nil +} + +// convertCostItems turns each item's list of {costKey, costValue} pairs into a map from key to value. +func convertCostItems(rawItems []rawCostItem) []schema.CostItem { + items := make([]schema.CostItem, 0, len(rawItems)) + for _, rawItem := range rawItems { + costs := make(map[string]float64, len(rawItem.Costs)) + for _, cost := range rawItem.Costs { + costs[cost.Key] = cost.Value + } + items = append(items, schema.CostItem{Name: rawItem.Name, Type: rawItem.Type, Costs: costs}) + } + return items +}