3b194ff2e8
Some checks failed
Build CLI with Embedded SQLite / build (arm64, aarch64-linux) (push) Waiting to run
Build CLI with Embedded SQLite / build (x86_64, x86_64-linux) (push) Waiting to run
Build CLI with Embedded SQLite / build-macos (arm64) (push) Waiting to run
Build CLI with Embedded SQLite / build-macos (x86_64) (push) Waiting to run
Security Scan / Security Analysis (push) Waiting to run
Security Scan / Native Library Security (push) Waiting to run
Checkout test / test (push) Successful in 6s
CI/CD Pipeline / Test (push) Failing after 1s
CI/CD Pipeline / Dev Compose Smoke Test (push) Has been skipped
CI/CD Pipeline / Build (push) Has been skipped
CI/CD Pipeline / Test Scripts (push) Has been skipped
CI/CD Pipeline / Test Native Libraries (push) Has been skipped
CI/CD Pipeline / GPU Golden Test Matrix (push) Has been skipped
Documentation / build-and-publish (push) Failing after 39s
CI/CD Pipeline / Docker Build (push) Has been skipped
- Surface GPUDetectionInfo from parseGPUCountFromConfig for detection metadata - Document FETCH_ML_TOTAL_CPU and FETCH_ML_GPU_SLOTS_PER_GPU env vars - Add debug logging for all env var overrides to stderr - Track config-layer auto-detection in GPUDetectionInfo.ConfigLayerAutoDetected - Add --include-all flag to artifact scanner (includeAll parameter) - Add AMD production mode enforcement (error in non-local mode) - Add GPU detector unit tests for env overrides and AMD aliasing
80 lines
2 KiB
Go
80 lines
2 KiB
Go
//go:build cgo && native_libs
|
|
// +build cgo,native_libs
|
|
|
|
package worker
|
|
|
|
// #cgo darwin LDFLAGS: -L${SRCDIR}/../../native/build -Wl,-rpath,${SRCDIR}/../../native/build -ldataset_hash
|
|
// #cgo linux LDFLAGS: -L${SRCDIR}/../../native/build -Wl,-rpath,${SRCDIR}/../../native/build -ldataset_hash -lnvml_gpu -lnvidia-ml
|
|
// #include "../../native/dataset_hash/dataset_hash.h"
|
|
// #include <stdlib.h>
|
|
import "C"
|
|
|
|
import (
|
|
"errors"
|
|
"log"
|
|
"runtime"
|
|
"sync"
|
|
"time"
|
|
"unsafe"
|
|
|
|
"github.com/jfraeys/fetch_ml/internal/manifest"
|
|
)
|
|
|
|
var (
|
|
hashCtx *C.fh_context_t
|
|
hashCtxOnce sync.Once
|
|
ctxInitTime time.Time
|
|
)
|
|
|
|
// getHashContext returns the native hash context, initializing it on first call.
|
|
// First call initializes C++ context (5-20ms) - subsequent calls reuse context.
|
|
func getHashContext() *C.fh_context_t {
|
|
hashCtxOnce.Do(func() {
|
|
start := time.Now()
|
|
hashCtx = C.fh_init(C.uint32_t(runtime.NumCPU()))
|
|
ctxInitTime = time.Now()
|
|
log.Printf("[native] hash context initialized: %v (threads: %d)",
|
|
time.Since(start), runtime.NumCPU())
|
|
})
|
|
return hashCtx
|
|
}
|
|
|
|
func dirOverallSHA256HexNative(root string) (string, error) {
|
|
ctx := getHashContext()
|
|
|
|
croot := C.CString(root)
|
|
defer C.free(unsafe.Pointer(croot))
|
|
|
|
result := C.fh_hash_directory_combined(ctx, croot)
|
|
if result == nil {
|
|
err := C.fh_last_error(ctx)
|
|
if err != nil {
|
|
return "", errors.New(C.GoString(err))
|
|
}
|
|
return "", errors.New("native hash failed")
|
|
}
|
|
defer C.fh_free_string(result)
|
|
|
|
return C.GoString(result), nil
|
|
}
|
|
|
|
func GetSIMDImplName() string {
|
|
return C.GoString(C.fh_get_simd_impl_name())
|
|
}
|
|
|
|
func HasSIMDSHA256() bool {
|
|
return C.fh_has_simd_sha256() == 1
|
|
}
|
|
|
|
func ScanArtifactsNative(runDir string) (*manifest.Artifacts, error) {
|
|
return ScanArtifacts(runDir, false)
|
|
}
|
|
|
|
func ExtractTarGzNative(archivePath, dstDir string) error {
|
|
return ExtractTarGz(archivePath, dstDir)
|
|
}
|
|
|
|
// DirOverallSHA256HexNative exports the native hash implementation for benchmarks.
|
|
func DirOverallSHA256HexNative(root string) (string, error) {
|
|
return dirOverallSHA256HexNative(root)
|
|
}
|