GopherProxy

	tmain.go: parse cpt data - ags-upload - Insert AGS files to a database
	git clone git://src.adamsgaard.dk/ags-upload
	Log
	Files
	Refs
	---
	commit d7aff58399bcb2d8e0573c9e027229bb9d0fd3f3
	parent 12b840e26e1e7437fea2415dc2794eb9a1c932cd
	Author: Anders Damsgaard <[email protected]>
	Date: Wed, 8 Oct 2025 11:40:06 +0200

	main.go: parse cpt data

	Diffstat:
	M cmd/main.go \| 171 ++++++++++++++++++++++-------…

	1 file changed, 123 insertions(+), 48 deletions(-)
	---
	diff --git a/cmd/main.go b/cmd/main.go
	t@@ -9,7 +9,8 @@ import (
	"net/http"
	"os"
	"strings"
	- "time"
	+ "strconv"
	+ //"time"

	"github.com/gin-gonic/gin"
	"gorm.io/driver/postgres"
	t@@ -26,11 +27,24 @@ type CptInfo struct {
	Contractor string // PROJ_CONT
	}

	-func ParseAGS(r io.Reader) (*CptInfo, error) {
	+type Cpt struct { // group SCPG - data
	+ ID uint `gorm:"primaryKey"`
	+ InfoId uint //foreign key from CptInfo
	+ LocationId string // LOCA_ID
	+ TestReference string // SCPG_TESN
	+ Depth float64 // SCPT_DPTH
	+ ConeRes float64 // SCPT_RES
	+ SideFric float64 // SCPT_FRES
	+ Pore1 float64 // SCPT_PWP1
	+ Pore2 float64 // SCPT_PWP2
	+ Pore3 float64 // SCPT_PWP3
	+ FrictionRatio float64 // SCPT_FRR
	+}

	+func ParseAGSProjectAndSCPT(r io.Reader) (*CptInfo, []Cpt, error) {
	norm, err := dos2unix(r)
	if err != nil {
	- return nil, fmt.Errorf("read: %w", err)
	+ return nil, nil, fmt.Errorf("read: %w", err)
	}

	cr := csv.NewReader(norm)
	t@@ -38,69 +52,113 @@ func ParseAGS(r io.Reader) (*CptInfo, error) {
	cr.LazyQuotes = true

	var (
	- inPROJ bool
	- headerIndex map[string]int
	+ curGroup string
	+ headersByGrp = map[string]map[string]int{} // GROUP -> header …
	+ project *CptInfo
	+ cpts []Cpt
	)

	+ get := func(group string, data []string, name string) string {
	+ hm := headersByGrp[group]
	+ if hm == nil {
	+ return ""
	+ }
	+ if idx, ok := hm[strings.ToUpper(name)]; ok && idx >= 0 && idx…
	+ return data[idx]
	+ }
	+ return ""
	+ }
	+ parseF64 := func(s string) float64 {
	+ if s == "" {
	+ return 0
	+ }
	+ // Optional: handle decimal commas
	+ s = strings.ReplaceAll(s, ",", ".")
	+ f, _ := strconv.ParseFloat(s, 64)
	+ return f
	+ }
	+
	for {
	rec, err := cr.Read()
	if err == io.EOF {
	break
	}
	if err != nil {
	- return nil, fmt.Errorf("csv: %w", err)
	+ return nil, nil, fmt.Errorf("csv: %w", err)
	}
	if len(rec) == 0 {
	continue
	}
	-
	for i := range rec {
	rec[i] = strings.TrimSpace(rec[i])
	+ // Some exporters put empty quotes => "" — leave as …
	}

	- switch strings.ToUpper(rec[0]) {
	+ tag := strings.ToUpper(rec[0])
	+ switch tag {
	case "GROUP":
	- inPROJ = len(rec) > 1 && strings.EqualFold(rec[1], "PR…
	- headerIndex = nil
	+ if len(rec) > 1 {
	+ curGroup = strings.ToUpper(strings.TrimSpace(r…
	+ } else {
	+ curGroup = ""
	+ }
	case "HEADING":
	- if !inPROJ {
	+ if curGroup == "" {
	continue
	}
	- headerIndex = make(map[string]int)
	+ m := make(map[string]int, len(rec)-1)
	for i := 1; i < len(rec); i++ {
	key := strings.ToUpper(strings.TrimSpace(rec[i…
	- headerIndex[key] = i - 1 // positions in the "…
	+ m[key] = i - 1 // position in DATA after skipp…
	}
	+ headersByGrp[curGroup] = m
	+
	case "DATA":
	- if !inPROJ \|\| headerIndex == nil {
	+ if curGroup == "" {
	continue
	}
	- data := rec[1:] // align with headerIndex positions
	+ data := rec[1:]

	- get := func(h string) string {
	- if idx, ok := headerIndex[strings.ToUpper(h)];…
	- return data[idx]
	+ switch curGroup {
	+ case "PROJ":
	+ if project != nil {
	+ // If multiple PROJ rows exist, keep t…
	+ continue
	+ }
	+ project = &CptInfo{
	+ SourceId: get("PROJ", data, "PROJ_ID…
	+ Name: get("PROJ", data, "PROJ_NA…
	+ Location: get("PROJ", data, "PROJ_LO…
	+ Client: get("PROJ", data, "PROJ_CL…
	+ Contractor: get("PROJ", data, "PROJ_CO…
	}
	- return ""
	- }

	- p := &CptInfo{
	- SourceId: get("PROJ_ID"),
	- Name: get("PROJ_NAME"),
	- Location: get("PROJ_LOC"),
	- Client: get("PROJ_CLNT"),
	- Contractor: get("PROJ_CONT"),
	+ case "SCPT":
	+ cpts = append(cpts, Cpt{
	+ LocationId: get("SCPT", data, "LOCA…
	+ TestReference: get("SCPT", data, "SCPG…
	+ Depth: parseF64(get("SCPT", da…
	+ ConeRes: parseF64(get("SCPT", da…
	+ SideFric: parseF64(get("SCPT", da…
	+ Pore1: parseF64(get("SCPT", da…
	+ Pore2: parseF64(get("SCPT", da…
	+ Pore3: parseF64(get("SCPT", da…
	+ FrictionRatio: parseF64(get("SCPT", da…
	+ })
	+ default:
	+ // ignore other groups for now
	}
	- return p, nil

	+ // ignore UNIT, TYPE, etc.
	default:
	continue
	}
	}

	- return nil, fmt.Errorf("no data found")
	+ return project, cpts, nil
	}

	+
	func dos2unix(r io.Reader) (io.Reader, error) {
	all, err := io.ReadAll(r)
	if err != nil {
	t@@ -135,44 +193,61 @@ func main() {
	log.Fatal(err)
	}

	+ if err := db.AutoMigrate(&Cpt{}); err != nil {
	+ log.Fatal(err)
	+ }
	+
	r := gin.Default()

	// ~32 MB file cap for multipart
	r.MaxMultipartMemory = 32 << 20

	r.POST("/ingest/ags", func(c *gin.Context) {
	- reader, cleanup, err := getAGSReader(c.Request)
	+ file, _, err := c.Request.FormFile("file")
	if err != nil {
	- c.String(http.StatusBadRequest, "upload error: %v", er…
	+ c.String(400, "missing multipart file: %v", err)
	return
	}
	- if cleanup != nil {
	- defer cleanup()
	- }
	+ defer file.Close()

	- p, err := ParseAGS(reader)
	+ proj, cpts, err := ParseAGSProjectAndSCPT(file)
	if err != nil {
	- c.String(http.StatusBadRequest, "parse error: %v", err)
	+ c.String(400, "parse error: %v", err)
	return
	}

	- err = db.
	- Where("source_id = ?", p.SourceId).
	- Assign(p).
	- FirstOrCreate(p).Error
	+ err = db.Transaction(func(tx *gorm.DB) error {
	+
	+ // Upsert project by SourceId (make SourceId unique if…
	+ if proj != nil {
	+ if err := tx.
	+ Where("source_id = ?", proj.SourceId).
	+ Assign(proj).
	+ FirstOrCreate(proj).Error; err != nil {
	+ return err
	+ }
	+ }
	+
	+ // If you later derive InfoId from a SC* info table, s…
	+ if len(cpts) > 0 {
	+ // Optional: add a foreign key to project if y…
	+ // for i := range cpts { cpts[i].ProjectID = p…
	+
	+ if err := tx.CreateInBatches(cpts, 2000).Error…
	+ return err
	+ }
	+ }
	+
	+ return nil
	+ })
	if err != nil {
	- c.String(http.StatusInternalServerError, "db error: %v…
	+ c.String(500, "db error: %v", err)
	return
	}

	- c.JSON(http.StatusCreated, gin.H{
	- "id": p.ID,
	- "sourceId": p.SourceId,
	- "name": p.Name,
	- "location": p.Location,
	- "client": p.Client,
	- "contractor": p.Contractor,
	- "savedAt": time.Now().Format(time.RFC3339),
	+ c.JSON(201, gin.H{
	+ "project": proj,
	+ "cpts": len(cpts),
	})
	})