/*
Store a public key certificate in PEM format into a file in the standard directory in the relish installation,
using standard file naming convention.
*/
func StorePublicKeyCert(entityType string, entityName string, publicKeyCertPEM string) (err error) {
fileName := entityType + "__" + entityName + "__public_key.pem"
keyDirPath := relishRuntimeLocation + "/keys/public"
path := keyDirPath + "/" + fileName
var perm os.FileMode = 0777
err = gos.MkdirAll(keyDirPath, perm)
if err != nil {
return
}
perm = 0666
err = gos.WriteFile(path, ([]byte)(publicKeyCertPEM), perm)
return
}
// path = "mydirectory"
// permissions = "rwxrw_r__" // or permissions = "764"
//
// err = mkdirAll path [permissions]
//
func mkdirAll(th InterpreterThread, objects []RObject) []RObject {
path := string(objects[0].(String))
permStr := "766"
if len(objects) > 1 {
permStr = string(objects[1].(String))
}
perm, errStr := getFilePermissions(permStr)
var err error
if errStr == "" {
err = gos.MkdirAll(path, perm)
}
if err != nil {
errStr = err.Error()
}
return []RObject{String(errStr)}
}
/*
Extracts contents of a zip file into the specified directory, which must exist and be writeable.
The directory path must not end with a "/".
The directory path must be expressed in unix-style path syntax, with "/" separators not "\" or "\\"
Always excludes __MACOSX folders from what it writes to the target directory tree.
*/
func ExtractZipFileContents(zipFileContents []byte, dirPath string) (err error) {
var perm os.FileMode = 0777
byteSliceReader := bytes.NewReader(zipFileContents)
var r *zip.Reader
r, err = zip.NewReader(byteSliceReader, int64(len(zipFileContents)))
if err != nil {
return
}
// Iterate through the files in the archive,
// copying their contents.
for _, f := range r.File {
fileInfo := f.FileHeader.FileInfo()
// if strings.Index(fileInfo.Name(),"__MACOSX") == 0 {
// replaced with following line to be Go1.2 compatible.
if strings.Index(f.FileHeader.Name, "__MACOSX") == 0 {
continue
}
if fileInfo.IsDir() {
Log(LOAD2_, "Directory %s:\n", f.Name)
err = gos.MkdirAll(dirPath+"/"+f.Name, perm)
if err != nil {
return
}
} else {
Log(LOAD2_, "Copying file %s:\n", f.Name)
var rc io.ReadCloser
rc, err = f.Open()
if err != nil {
return
}
slashPos := strings.LastIndex(f.Name, "/")
if slashPos != -1 {
relativeDirPath := f.Name[:slashPos]
err = gos.MkdirAll(dirPath+"/"+relativeDirPath, perm)
if err != nil {
return
}
}
var outFile *os.File
outFile, err = gos.Create(dirPath + "/" + f.Name)
if err != nil {
return
}
_, err = io.Copy(outFile, rc)
if err != nil {
return
}
rc.Close()
outFile.Close()
Logln(LOAD2_)
}
}
return
}
/*
Creates a directory tree for a new relish artifact.
Creates template versions of some of the files you will need.
projectType can currently be "webapp" or ""
*/
func initProject(relishRoot, projectPath, projectType string) (err error) {
artifactDir := relishRoot + "/artifacts/" + projectPath
slashPos := strings.Index(projectPath, "/")
origin := projectPath[0:slashPos]
artifact := projectPath[slashPos+1:]
metadataFilePath := artifactDir + "/metadata.txt"
date := time.Now().Format("2006/01/02")
err = gos.MkdirAll(artifactDir, 0777)
if err != nil {
return
}
mainPackageDir := artifactDir + "/v0.1.0/src/main"
err = gos.MkdirAll(mainPackageDir, 0777)
if err != nil {
return
}
mainFilePath := mainPackageDir + "/main.rel"
if projectType == "webapp" {
metadata := fmt.Sprintf(WEB_APP_METADATA_FILE, date, origin, artifact)
err = gos.WriteFile(metadataFilePath, ([]byte)(metadata), 0777)
if err != nil {
return
}
// TODO Create web package directory with template index.html and dialog.rel and static/ with styles and an image and a static html file.
webPackageDir := artifactDir + "/v0.1.0/src/web"
err = gos.MkdirAll(webPackageDir, 0777)
if err != nil {
return
}
indexPath := webPackageDir + "/index.html"
err = gos.WriteFile(indexPath, ([]byte)(INDEX_HTML_FILE), 0777)
if err != nil {
return
}
visitCountPath := webPackageDir + "/visit_count.html"
err = gos.WriteFile(visitCountPath, ([]byte)(VISIT_COUNT_HTML_FILE), 0777)
if err != nil {
return
}
gotItPath := webPackageDir + "/got_it.html"
err = gos.WriteFile(gotItPath, ([]byte)(GOT_IT_HTML_FILE), 0777)
if err != nil {
return
}
tryAgainPath := webPackageDir + "/try_again.html"
err = gos.WriteFile(tryAgainPath, ([]byte)(TRY_AGAIN_HTML_FILE), 0777)
if err != nil {
return
}
dialogPath := webPackageDir + "/dialog.rel"
dialogContent := fmt.Sprintf(DIALOG_FILE, origin, artifact)
err = gos.WriteFile(dialogPath, ([]byte)(dialogContent), 0777)
if err != nil {
return
}
stylesDir := webPackageDir + "/static/styles"
err = gos.MkdirAll(stylesDir, 0777)
if err != nil {
return
}
cssPath := stylesDir + "/default.css"
err = gos.WriteFile(cssPath, ([]byte)(CSS_FILE), 0777)
if err != nil {
return
}
staticPath := webPackageDir + "/static/example.html"
err = gos.WriteFile(staticPath, ([]byte)(STATIC_HTML_FILE), 0777)
if err != nil {
return
}
imgDir := webPackageDir + "/static/img"
err = gos.MkdirAll(imgDir, 0777)
if err != nil {
return
}
imagePath := imgDir + "/relish_logo_small.png"
var imageBytes []byte
imageBytes, err = base64.URLEncoding.DecodeString(RELISH_LOGO_PNG)
if err != nil {
return
}
err = gos.WriteFile(imagePath, imageBytes, 0777)
if err != nil {
return
}
mainContent := fmt.Sprintf(WEB_APP_MAIN_PROGRAM_FILE, origin, artifact, artifact, origin, artifact)
err = gos.WriteFile(mainFilePath, ([]byte)(mainContent), 0777)
if err != nil {
return
}
// TODO create index.html, dialog.rel,default.css,an image??,a static html file.
} else if projectType == "" {
metadata := fmt.Sprintf(APP_METADATA_FILE, date, origin, artifact)
err = gos.WriteFile(metadataFilePath, ([]byte)(metadata), 0777)
if err != nil {
return
}
mainContent := fmt.Sprintf(APP_MAIN_PROGRAM_FILE, origin, artifact, artifact, origin, artifact)
err = gos.WriteFile(mainFilePath, ([]byte)(mainContent), 0777)
if err != nil {
return
}
} else {
err = fmt.Errorf("Unrecognized relish project type '%s'.", projectType)
}
return
}
func main() {
var loggingLevel int
var webListeningPort int
var tlsWebListeningPort int
var shareListeningPort int // port on which source code will be shared by http
var sharedCodeOnly bool // do not use local artifacts - only those in shared directory.
var explorerListeningPort int // port on which data explorer_api web service will be served.
var runningArtifactMustBeFromShared bool
var dbName string
var cpuprofile string
var publish bool
var quiet bool
var projectPath string
// var gcIntervalSeconds int
//var fset = token.NewFileSet()
flag.IntVar(&loggingLevel, "log", 0, "The logging level: 0 is least verbose, 2 most")
flag.IntVar(&webListeningPort, "web", 0, "The http listening port - if not supplied, does not listen for http requests")
flag.IntVar(&tlsWebListeningPort, "tls", 0, "The https listening port - if not supplied, does not listen for https requests")
flag.IntVar(&explorerListeningPort, "explore", 0, "The explorer_api web service listening port - if supplied, the data explorer tool can connect to this program on this port")
flag.StringVar(&dbName, "db", "db1", "The database name. A SQLITE database file called <name>.db will be created/used in artifact data directory")
flag.BoolVar(&sharedCodeOnly, "shared", false, "Use shared version of all artifacts - ignore local/dev copy of artifacts")
flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")
flag.IntVar(&shareListeningPort, "share", 0, "The code sharing http listening port - if not supplied, does not listen for source code sharing http requests")
flag.BoolVar(&publish, "publish", false, "artifactpath version - copy specified version of artifact to shared/relish/artifacts")
flag.BoolVar(&quiet, "quiet", false, "do not show package loading info or interpreter version in program output")
flag.StringVar(&projectPath, "init", "", "<artifactpath> [webapp] - create directory tree and template files for a relish software project")
flag.IntVar(¶ms.GcIntervalSeconds, "gc", params.GcIntervalSeconds, "The garbage collection check interval (seconds): defaults to 20")
flag.IntVar(¶ms.DbMaxConnections, "pool", params.DbMaxConnections, "Maximum number of db connections to open with the database: defaults to 1")
flag.IntVar(¶ms.DbMaxReadConnections, "rpool", params.DbMaxReadConnections, "Maximum number of read-only db connections to open with the database")
flag.IntVar(¶ms.DbMaxWriteConnections, "wpool", params.DbMaxWriteConnections, "Maximum number of writeable db connections to open with the database")
flag.Parse()
pathParts := flag.Args() // full path to package, or originAndArtifact and path to package
// (or originAndArtifact and version number if -publish)
if cpuprofile != "" {
f, err := gos.Create(cpuprofile)
if err != nil {
fmt.Println(err)
return
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
dbg.InitLogging(int32(loggingLevel))
//relish.InitRuntime("relish.db")
// if ! publish {
// builtin.InitBuiltinFunctions()
// }
var g *generator.Generator
var relishRoot string // This actually has to be the root of the runtime environment
// i.e. /opt/relish if this is a binary distribution,
// or /opt/relish/rt if this is a source distribution
workingDirectory, err := gos.Getwd()
if err != nil {
fmt.Printf("Cannot determine working directory: %s\n", err)
}
var originAndArtifact string
var version string
var packagePath string
isSubDir := false
isSourceDist := false
relishIndexInWd := strings.Index(workingDirectory, "/relish/")
if relishIndexInWd > -1 {
isSubDir = true
} else {
relishIndexInWd = strings.Index(workingDirectory, "/relish")
if relishIndexInWd == -1 || relishIndexInWd != len(workingDirectory)-7 {
if projectPath != "" { // Must be creating a relish dir under a new project dir
relishRoot = workingDirectory + "/relish"
err = gos.MkdirAll(relishRoot, 0777)
if err != nil {
fmt.Printf("Error making relish project directory %s: %s\n", relishRoot, err)
return
}
} else {
fmt.Printf("relish command must be run from within a relish directory tree.\n")
return
}
}
}
if relishRoot == "" {
relishRoot = workingDirectory[:relishIndexInWd+7]
}
_, err = gos.Stat(relishRoot + "/rt")
if err == nil {
relishRoot += "/rt"
isSourceDist = true
} else if !os.IsNotExist(err) {
fmt.Printf("Can't stat '%s' : %v\n", relishRoot+"/rt", err)
return
}
// relishRoot is now established
if projectPath != "" {
if len(pathParts) < 1 {
err = initProject(relishRoot, projectPath, "")
if err == nil {
fmt.Printf("\nCreated relish project template\n%s/artifacts/%s\n", relishRoot, projectPath)
fmt.Printf("\nTo run your project template's dummy main program, relish %s\n\n", projectPath)
} else {
fmt.Printf("\nError initializing project %s: %s\n", projectPath, err)
}
} else {
projectType := pathParts[0]
err = initProject(relishRoot, projectPath, projectType)
if err == nil {
fmt.Printf("\nCreated relish web-app project template\n%s/artifacts/%s\n", relishRoot, projectPath)
fmt.Printf("\nTo run the web-app, relish -web 8080 %s\n", projectPath)
fmt.Printf("Then enter localhost:8080 into your browser's address bar to view the web app.\n\n")
} else {
fmt.Printf("\nError initializing project %s: %s\n", projectPath, err)
}
}
return
}
if isSubDir { // See where we are more specifically
if isSourceDist {
idx := strings.Index(workingDirectory, "/rt/shared")
if idx > -1 {
runningArtifactMustBeFromShared = true
}
} else {
idx := strings.Index(workingDirectory, "/relish/shared")
if idx > -1 {
runningArtifactMustBeFromShared = true
}
}
if !publish {
// See if the current directory is a particular artifact version directory,
// or even is a particular package directory.
originPos := strings.Index(workingDirectory, "/artifacts/") + 11
if originPos == 10 {
originPos = strings.Index(workingDirectory, "/replicas/") + 10
}
if originPos >= 10 {
match := reVersionedPackage.FindStringSubmatchIndex(workingDirectory)
if match != nil {
version = workingDirectory[match[2]:match[3]]
originAndArtifact = workingDirectory[originPos:match[0]]
packagePath = workingDirectory[match[3]+1:]
} else {
match := reVersionAtEnd.FindStringSubmatch(workingDirectory)
if match != nil {
version = match[1]
originAndArtifact = workingDirectory[originPos : len(workingDirectory)-len(version)-2]
}
}
}
}
}
if !publish {
builtin.InitBuiltinFunctions(relishRoot)
}
crypto_util.SetRelishRuntimeLocation(relishRoot) // So that keys can be fetched.
if publish {
if len(pathParts) < 2 {
fmt.Println("Usage (example): relish -publish someorigin.com2013/artifact_name 1.0.23")
return
}
originAndArtifact = pathParts[0]
version = pathParts[1]
if strings.HasSuffix(originAndArtifact, "/") { // Strip trailing / if present
originAndArtifact = originAndArtifact[:len(originAndArtifact)-1]
}
err = global_publisher.PublishSourceCode(relishRoot, originAndArtifact, version)
if err != nil {
fmt.Println(err)
}
return
}
sourceCodeShareDir := ""
if shareListeningPort != 0 {
// sourceCodeShareDir hould be the "relish/shared"
// or "relish/rt/shared" of "relish/4production/shared" or "relish/rt/4production/shared" directory.
sourceCodeShareDir = relishRoot + "/shared"
}
onlyCodeSharing := (shareListeningPort != 0 && webListeningPort == 0 && tlsWebListeningPort == 0)
if onlyCodeSharing {
if shareListeningPort < 1024 && shareListeningPort != 80 {
fmt.Println("Error: The source-code sharing port must be 80 or > 1023 (8421 is the standard if using a high port)")
return
}
web.ListenAndServeSourceCode(shareListeningPort, sourceCodeShareDir)
// The previous call should wait on incoming socket connections on the specified port, and never return.
// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
// have permission to listen on a < 1024 port like port 80.
fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
"Did you setup to give relish permission to bind to privileged ports?\n"+
"or is another process already listening on this port?\n", shareListeningPort)
return
}
var loader = global_loader.NewLoader(relishRoot, sharedCodeOnly, dbName+".db", quiet)
if originAndArtifact == "" {
if len(pathParts) == 3 { // originAndArtifact version packagePath
originAndArtifact = pathParts[0]
version = pathParts[1]
packagePath = pathParts[2]
} else if len(pathParts) == 2 { // originAndArtifact packagePath
originAndArtifact = pathParts[0]
packagePathOrVersion := pathParts[1]
// Determine if is a version by regexp or a package path has been supplied
version = reVersion.FindString(packagePathOrVersion)
if version == "" {
packagePath = packagePathOrVersion
}
} else if shareListeningPort == 0 || webListeningPort != 0 || tlsWebListeningPort != 0 {
if len(pathParts) != 1 {
fmt.Println("Usage: relish [-web 80] originAndArtifact [version] [path/to/package]\n# package path defaults to main")
return
}
originAndArtifact = pathParts[0]
}
} else if packagePath == "" {
if len(pathParts) == 1 {
packagePath = pathParts[0]
} else if len(pathParts) > 1 {
fmt.Println("Usage (when in an artifact version directory): relish [-web 80] [path/to/package]\n# package path defaults to main")
return
}
} else { // both originAndArtifact and packagePath are defined (non "")
if len(pathParts) != 0 {
fmt.Println("Usage (when in a package directory): relish [-web 80]")
return
}
}
if strings.HasSuffix(originAndArtifact, "/") { // Strip trailing / if present
originAndArtifact = originAndArtifact[:len(originAndArtifact)-1]
}
if strings.HasSuffix(packagePath, "/") { // Strip trailing / if present
packagePath = packagePath[:len(packagePath)-1]
}
if packagePath == "" {
packagePath = "main" // substitute a default.
}
fullPackagePath := fmt.Sprintf("%s/v%s/pkg/%s", originAndArtifact, version, packagePath)
fullUnversionedPackagePath := fmt.Sprintf("%s/pkg/%s", originAndArtifact, packagePath)
g, err = loader.LoadPackage(originAndArtifact, version, packagePath, runningArtifactMustBeFromShared)
if err != nil {
if version == "" {
fmt.Printf("Error loading package %s from current version of %s: %v\n", packagePath, originAndArtifact, err)
} else {
fmt.Printf("Error loading package %s: %v\n", fullPackagePath, err)
}
return
}
g.Interp.SetRunningArtifact(originAndArtifact)
g.Interp.SetPackageLoader(loader)
// TODO the following rather twisty logic (from here to end of main method) could be straightened out.
// One of its purposes is to ensure that the last http listener is run in this goroutine rather
// than in a background one. And there can be different numbers of listeners...
// Count how many separate listeners there will be.
numListeners := 0
numListening := 0 // how many of those are already listening?
if webListeningPort != 0 {
numListeners += 1
if shareListeningPort != 0 && shareListeningPort != webListeningPort {
numListeners += 1
}
}
if tlsWebListeningPort != 0 {
numListeners += 1
}
if explorerListeningPort != 0 {
numListeners += 1
}
// end counting listeners
// check for disallowed port numbers, and if not, load the packages needed for web app serving
if webListeningPort != 0 {
if webListeningPort < 1024 && webListeningPort != 80 {
fmt.Println("Error: The web listening port must be 80 or > 1023")
return
}
if shareListeningPort != webListeningPort && shareListeningPort != 0 && shareListeningPort < 1024 && shareListeningPort != 80 {
fmt.Println("Error: The source-code sharing port must be 80 or > 1023 (8421 is the standard if using a high port)")
return
}
}
if tlsWebListeningPort != 0 {
if tlsWebListeningPort < 1024 && tlsWebListeningPort != 443 {
fmt.Println("Error: The tls web listening port must be 443 or > 1023")
return
}
}
if webListeningPort != 0 || tlsWebListeningPort != 0 {
err = loader.LoadWebPackages(originAndArtifact, version, runningArtifactMustBeFromShared)
if err != nil {
if version == "" {
fmt.Printf("Error loading web packages from current version of %s: %v\n", originAndArtifact, err)
} else {
fmt.Printf("Error loading web packages from version %s of %s: %v\n", version, originAndArtifact, err)
}
return
}
}
// check for disallowed port numbers, and if not, load the package needed for explorer_api web service serving
if explorerListeningPort != 0 {
if explorerListeningPort < 1024 && explorerListeningPort != 80 {
fmt.Println("Error: The explorer listening port must be 80 or > 1023")
return
}
explorerApiOriginAndArtifact := "shared.relish.pl2012/explorer_api"
explorerApiPackagePath := "web"
_, err = loader.LoadPackage(explorerApiOriginAndArtifact, "", explorerApiPackagePath, false)
if err != nil {
fmt.Printf("Error loading package %s from current version of %s: %v\n",
explorerApiPackagePath,
explorerApiOriginAndArtifact,
err)
return
}
}
if numListeners > 0 { // If we'll be listening for http requests, run main in a background goroutine.
go g.Interp.RunMain(fullUnversionedPackagePath, quiet)
}
if webListeningPort != 0 || tlsWebListeningPort != 0 {
web.SetWebPackageSrcDirPath(loader.PackageSrcDirPath(originAndArtifact + "/pkg/web"))
if webListeningPort != 0 {
if shareListeningPort == webListeningPort {
numListening += 1
if numListening == numListeners {
web.ListenAndServe(webListeningPort, sourceCodeShareDir)
// The previous call should wait on incoming socket connections on the specified port, and never return.
// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
// have permission to listen on a < 1024 port like port 80.
fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
"Did you setup to give relish permission to bind to privileged ports?\n"+
"or is another process already listening on this port?\n", webListeningPort)
return
} else {
go web.ListenAndServe(webListeningPort, sourceCodeShareDir)
}
} else {
if shareListeningPort != 0 {
numListening += 1
go web.ListenAndServeSourceCode(shareListeningPort, sourceCodeShareDir)
}
numListening += 1
if numListening == numListeners {
web.ListenAndServe(webListeningPort, "")
// The previous call should wait on incoming socket connections on the specified port, and never return.
// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
// have permission to listen on a < 1024 port like port 80.
fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
"Did you setup to give relish permission to bind to privileged ports?\n"+
"or is another process already listening on this port?\n", webListeningPort)
return
} else {
go web.ListenAndServe(webListeningPort, "")
}
}
}
if tlsWebListeningPort != 0 {
tlsCertPath, tlsKeyPath, err := crypto_util.GetTLSwebServerCertAndKeyFilePaths()
if err != nil {
fmt.Printf("Error starting TLS web listener: %s\n", err)
return
}
numListening += 1
if numListening == numListeners {
web.ListenAndServeTLS(tlsWebListeningPort, tlsCertPath, tlsKeyPath)
// The previous call should wait on incoming socket connections on the specified port, and never return.
// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
// have permission to listen on a < 1024 port like port 80.
fmt.Printf("Error: Could not bind to port %d to listen for https connections.\n"+
"Did you setup to give relish permission to bind to privileged ports?\n"+
"or is another process already listening on this port?\n", tlsWebListeningPort)
return
} else {
go web.ListenAndServeTLS(tlsWebListeningPort, tlsCertPath, tlsKeyPath)
}
}
}
if explorerListeningPort != 0 {
web.ListenAndServeExplorerApi(explorerListeningPort)
// The previous call should wait on incoming socket connections on the specified port, and never return.
// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
// have permission to listen on a < 1024 port like port 80.
fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
"Did you setup to give relish permission to bind to privileged ports?\n"+
"or is another process already listening on this port?\n", explorerListeningPort)
return
}
// This will only be reached if numListeners == 0 or there is an error starting listeners.
// but don't want to re-run main if there was an error starting listeners.
if numListeners == 0 {
g.Interp.RunMain(fullUnversionedPackagePath, quiet)
}
}
/*
Fetches and, if newer, installs metadata.txt file in shared/relish/replicas directory.
*/
func fetchArtifactMetadata(hostUrl string, originAndArtifactPath string, localMetadataDate string, existingSharedCurrentVersion string, sharedArtifactMetadataFilePath string) (currentVersion string, err error) {
slashPos := strings.Index(originAndArtifactPath, "/")
originDomain := originAndArtifactPath[0 : slashPos-4]
hostDomain := hostUrl[7:]
var dir string
if originDomain == hostDomain || strings.HasPrefix(hostDomain, "localhost") {
dir = "artifacts"
} else {
dir = "replicas"
}
url := hostUrl + "/relish/" + dir + "/" + originAndArtifactPath + "/metadata.txt"
var resp *http.Response
var body []byte
//resp, err = http.Get(url)
resp, err = artifactLoaderClient.Get(url)
if err != nil {
return
}
defer resp.Body.Close()
if resp.StatusCode == 404 { // HTTP Page/Resource Not Found
err = fmt.Errorf("%s not found at servers tried.", originAndArtifactPath)
return
} else if resp.StatusCode >= 400 { // Other HTTP error
err = fmt.Errorf("%s not found or HTTP errors at servers tried.", originAndArtifactPath)
return
}
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
return
}
var remoteCurrentVersion string
var remoteMetadataDate string
remoteCurrentVersion, remoteMetadataDate, err = readCurrentVersion(body, url)
if err != nil {
return
}
if remoteMetadataDate > localMetadataDate {
currentVersion = remoteCurrentVersion
// Create local directory for replica artifact, if necessary
var perm os.FileMode = 0777
artifactReplicaDir := sharedArtifactMetadataFilePath[:len(sharedArtifactMetadataFilePath)-len("/metadata.txt")]
err = gos.MkdirAll(artifactReplicaDir, perm)
if err != nil {
return
}
// Write body to local shared metadata.txt file
err = gos.WriteFile(sharedArtifactMetadataFilePath, body, perm)
if err != nil {
return
}
} else {
currentVersion = existingSharedCurrentVersion
}
return
}
/*
Loads the code from the specified package into the runtime.
Recurses to load packages which the argument package depends on.
TODO artifact version preferences are NOT being handled correctly yet.
What should happen is that all artifacts whose versions "ARE" recursively preferred should have their version preferences
consulted and respected first (closer to top of load-tree has preference priority) and then and only then should
a single no-version-preferred artifact be loaded by asking what its latest version is, then re-visit if that constrains other
formerly no-version-preferred ones, then load newly preferred versions, then load the next no-version-preferred artifact.
Currently, an artifact version may be loaded in a no-version-preferred way, because the current load-tree descent path does
not prefer a version, but it could be that a subsequently loaded artifact somewhere else in the load-tree COULD express
a preference for a different version of the artifact, but its preference is consulted too late, after the artifact is already loaded.
Handles searching first in a local (private) artifacts repository (directory tree) then a shared artifacts repository,
but only if a directive is not in effect to load from shared only, and only if another package from
the same artifact has not already been loaded, because in that case, the local (private) or shared decision has already
been made and must apply to subsequent packages loaded from the same artifact.
If not found locally, tries to load from the Internet (at several standard locations).
TODO signed-code integrity checks
*/
func (ldr *Loader) LoadPackage(originAndArtifactPath string, version string, packagePath string, mustBeFromShared bool) (gen *generator.Generator, err error) {
if Logging(PARSE_) {
parserDebugMode |= parser.Trace
}
// First, see if the package is already loaded. If so, return
packageIdentifier := originAndArtifactPath + "/pkg/" + packagePath
beingLoaded := ldr.PackagesBeingLoaded[packageIdentifier]
if beingLoaded {
err = fmt.Errorf("Package dependency loop. Package '%s' is also present in the tree of packages it imports.", packageIdentifier)
return
}
loadedVersion, found := ldr.LoadedPackages[packageIdentifier]
if found {
if loadedVersion != version && version != "" {
err = fmt.Errorf("Can't load version %s of '%s' since version %s is already loaded into runtime.", version, packageIdentifier, loadedVersion)
}
return
}
localArtifactMetadataFilePath := ldr.RelishRuntimeLocation + "/artifacts/" + originAndArtifactPath + "/metadata.txt"
sharedArtifactMetadataFilePath := ldr.RelishRuntimeLocation + "/shared/relish/artifacts/" + originAndArtifactPath + "/metadata.txt"
sharedReplicaMetadataFilePath := ldr.RelishRuntimeLocation + "/shared/relish/replicas/" + originAndArtifactPath + "/metadata.txt"
// Current version of artifact according to shared artifact metadata found in this relish directory tree.
sharedCurrentVersion := ""
// Date of the artifact metadata that is being relied on for the package load.
metadataDate := ""
ldr.PackagesBeingLoaded[packageIdentifier] = true
if !ldr.quiet {
Log(ALWAYS_, "Loading package %s\n", packageIdentifier)
}
Log(LOAD2_, "LoadPackage: ldr.RelishRuntimeLocation=%s\n", ldr.RelishRuntimeLocation)
Log(LOAD2_, "LoadPackage: originAndArtifactPath=%s\n", originAndArtifactPath)
Log(LOAD2_, "LoadPackage: version=%s\n", version)
Log(LOAD2_, "LoadPackage: packagePath=%s\n", packagePath)
mustBeFromShared = mustBeFromShared || ldr.SharedCodeOnly // Set whether will consider local code for this package.
var mustBeFromLocal bool // We may end up constrained to load from local artifact.
// Package is not loaded. But see if any other packages from the same artifact are loaded.
// If so, make sure they don't have an incompatible version.
var artifactAlreadyLoaded bool // if true, at least one package from the currently-being-loaded artifact has already been loade.
// This means the needed version of the artifact and the artifacts it depends on have already
// been loaded from built.txt into LoadedArtifacts map.
var artifactKnownToBeLocal bool // if artifact is loaded or being loaded, is it loaded from local
var artifactKnownToBePublished bool // if artifact is loaded or being loaded, is it loaded from shared
var artifactKnownToBeReplica bool // if artifact is loaded or being loaded, is it downloaded and loaded from shared/replicas
loadedVersion, artifactAlreadyLoaded = ldr.LoadedArtifacts[originAndArtifactPath]
if artifactAlreadyLoaded {
if loadedVersion != version && version != "" {
err = fmt.Errorf("Can't load package '%s' from version %s of '%s'. Another package from version %s of the artifact is already loaded into runtime.", packagePath, version, originAndArtifactPath, loadedVersion)
return
}
artifactKnownToBeLocal = ldr.LoadedArtifactKnownToBeLocal[originAndArtifactPath]
artifactKnownToBePublished = ldr.LoadedArtifactKnownToBePublished[originAndArtifactPath]
artifactKnownToBeReplica = ldr.LoadedArtifactKnownToBeReplica[originAndArtifactPath]
mustBeFromShared = mustBeFromShared || artifactKnownToBePublished || artifactKnownToBeReplica // Set whether will consider local code for this package.
Log(LOAD2_, "%s %s mustBeFromShared=%v\n", originAndArtifactPath, packagePath, mustBeFromShared)
if artifactKnownToBeLocal {
if mustBeFromShared {
// This should never happen I think. Check anyway.
err = fmt.Errorf("Can't load package '%s' from shared artifact '%s'. Another package from the local copy of the artifact is already loaded into runtime.", packagePath, originAndArtifactPath)
return
} else {
mustBeFromLocal = true
}
}
}
// Now try to load the package from local file system.
// If allowed, need to try twice, trying to read from
// local artifacts dir tree, then if not found from shared artifacts dir tree.
//// If no version has been specified, but some version of the artifact exists in local disk,
//// the first thing to do is to read the metadata.txt of the local artifact, and set the version number desired
//// to the current version as specified by the local artifact copy. Note this could be out of date, but we need a
//// different command to go check if there is a later version of the artifact out there and download it.
//// If we can find a metadata.txt file for the artifact locally, set the version with it.
if version == "" {
if !mustBeFromShared {
version, metadataDate, err = ldr.readMetadataFile(localArtifactMetadataFilePath)
if err != nil {
return
}
}
if version == "" {
if mustBeFromLocal {
// We already loaded a package from the local artifact, but somehow the local artifact is no longer there on filesystem.
// This should never happen if everything is being loaded at once at beginning of run. Check anyway.
err = fmt.Errorf("Can't load package '%s' from local artifact '%s'. Local artifact not found.", packagePath, originAndArtifactPath)
return
}
version, metadataDate, err = ldr.readMetadataFile(sharedArtifactMetadataFilePath)
if err != nil {
return
}
if version == "" {
version, metadataDate, err = ldr.readMetadataFile(sharedReplicaMetadataFilePath)
if err != nil {
return
}
if version != "" {
artifactKnownToBeReplica = true
}
} else {
artifactKnownToBePublished = true
}
sharedCurrentVersion = version
}
}
// stat the artifact version dir to see if the version of the artifact exists in the filesystem.
//
// try local then shared artifacts and replicas dir trees as allowed by constraints so far
artifactVersionDirFound := false
var artifactVersionDir string
if version != "" {
versionStr := "/v" + version
if !mustBeFromShared {
// try local artifacts dir tree
artifactVersionDir = ldr.RelishRuntimeLocation + "/artifacts/" + originAndArtifactPath + versionStr
_, statErr := gos.Stat(artifactVersionDir)
if statErr != nil {
if !os.IsNotExist(statErr) {
err = fmt.Errorf("Can't stat relish artifact version directory '%s': %v\n", artifactVersionDir, statErr)
return
}
} else {
artifactVersionDirFound = true
mustBeFromLocal = true // locked to local (private) artifact now
artifactKnownToBeLocal = true
}
}
if !artifactVersionDirFound {
// this version not found in local artifacts dir tree
// try published shared artifacts dir tree
artifactVersionDir = ldr.RelishRuntimeLocation + "/shared/relish/artifacts/" + originAndArtifactPath + versionStr
_, statErr := gos.Stat(artifactVersionDir)
if statErr != nil {
if !os.IsNotExist(statErr) {
err = fmt.Errorf("Can't stat relish artifact version directory '%s': %v\n", artifactVersionDir, statErr)
return
}
} else {
artifactVersionDirFound = true
mustBeFromShared = true // locked to shared artifact now
artifactKnownToBePublished = true
}
}
if !artifactVersionDirFound {
// this version not found in local artifacts dir tree or published shared artifacts dir tree
// try downloaded shared replicas dir tree
artifactVersionDir = ldr.RelishRuntimeLocation + "/shared/relish/replicas/" + originAndArtifactPath + versionStr
_, statErr := gos.Stat(artifactVersionDir)
if statErr != nil {
if !os.IsNotExist(statErr) {
err = fmt.Errorf("Can't stat relish artifact version directory '%s': %v\n", artifactVersionDir, statErr)
return
}
} else {
artifactVersionDirFound = true
mustBeFromShared = true // locked to shared artifact now
artifactKnownToBeReplica = true
}
}
}
if !artifactVersionDirFound {
// Have not found the artifact version locally. Fetch it from the Internet.
Log(LOAD2_, "artifact version |%s| not found locally\n", version)
Log(LOAD2_, "localArtifactMetadataFilePath=%s\n", localArtifactMetadataFilePath)
// TODO Need this path in order to install or update the artifact metadata file from remote, if there is none locally
// or if the remote one is more recent.
//
// artifactMetadataFilePath := ldr.RelishRuntimeLocation + "/shared/relish/artifacts/" + originAndArtifactPath + "/metadata.txt"
// Note: We will always be fetching into the shared artifacts directory tree.
// If programmer wants to copy an artifact version into the local artifacts directory tree to develop/modify it,
// they must currently do that copy separately manually.
var zipFileContents []byte
if sharedCurrentVersion == "" {
sharedCurrentVersion, metadataDate, err = ldr.readMetadataFile(sharedReplicaMetadataFilePath)
}
// replaced by stuff below
// defaultHostURL := ldr.DefaultCodeHost(originAndArtifactPath)
// hostURL := defaultHostURL
//
// REVIEW THIS COMMENT
// Note: TODO The correct order to do things is to load the metadata.txt file from the default host
// (if possible) then to search for secondary hosts to get the version zip file, selecting
// one AT RANDOM,
// then if all of those (some number of) mirrors fail, get it from the default host
// Also, use port 80 then 8421.
// TODO: RE: SERVER SEARCH ORDER
// We really ought to consider using a different order of servers tried
// for fetching artifact source-code zip files than the order we use for trying to find
// the smaller artifact metadata.txt files. Specifically, it is better to find
// metadata.txt files at servers owned or controlled by the origin, because the metadata.txt
// file will be up to date. shared.relish.pl is next best for that consideration.
// However, from a performance (load sharing when scaled) perspective, it is better to
// download the actual source code zip files from randomly found replica servers or
// secondary general repositories.
hostURLs := ldr.NonSearchedCodeHostURLs(originAndArtifactPath, "") // NEW
// If we did not have the metadata file on filesystem before, or if remote metadata file is newer,
// we should download and cache the metadata.txt file from the remote repository.
//
// Then, if we do not have a specified version yet, we should set version # from that,
var currentVersion string
var usingCentralRepo bool = false // whether getting code from http://shared.relish.pl
var hostURL string
for _, hostURL = range hostURLs {
// Read remote metadata file. Store it locally if its date is >= the local shared artifact metadata date.
currentVersion, err = fetchArtifactMetadata(hostURL, originAndArtifactPath, metadataDate, sharedCurrentVersion, sharedReplicaMetadataFilePath)
if err == nil {
break
} else if currentVersion != "" {
return // Inability to create or write local metadata file. Bad error.
}
}
if currentVersion == "" {
// TODO Now try google search
// Should really do a Google search for metadata found anywhere (except shared.relish.pl) now,
// so as to limit load and single point of failure on shared.relish.pl.
/*
hostURLs,err = ldr.FindSecondaryCodeHosts(originAndArtifactPath, hostURLs)
if err != nil {
return
}
for _,hostURL = range hostURLs {
// Read remote metadata file. Store it locally if its date is >= the local shared artifact metadata date.
currentVersion, err = fetchArtifactMetadata(hostURL, originAndArtifactPath, metadataDate, sharedCurrentVersion, sharedReplicaMetadataFilePath)
if err == nil {
break
} else if currentVersion != "" {
return // Inability to create or write local metadata file. Bad error.
}
}
*/
//
// Only if trying other replica sites fails should shared.relish.pl be tried.
//
// However, for now, we're going straight to trying shared.relish.pl, because Google searching and
// signed-metadata verification and signed-code verification aren't implemented yet.
}
if currentVersion == "" {
// Now try shared.relish.pl
usingCentralRepo = true
hostURL = "http://shared.relish.pl"
// Read remote metadata file. Store it locally if its date is >= the local shared artifact metadata date.
currentVersion, err = fetchArtifactMetadata(hostURL, originAndArtifactPath, metadataDate, sharedCurrentVersion, sharedReplicaMetadataFilePath)
if err != nil {
return // We really couldn't find and download metadata for this artifact anywhere we looked. Too bad.
}
}
// metadataHostURL := hostURL // If we need to keep track of where we got the metadata from.
if usingCentralRepo {
hostURLs = []string{hostURL}
} else {
hostURLs = ldr.NonSearchedCodeHostURLs(originAndArtifactPath, hostURL) // NEW
}
if version == "" {
version = currentVersion
}
// Version must now be a proper version number string, not ""
var zipFileName string
for _, hostURL = range hostURLs {
zipFileContents, zipFileName, err = fetchArtifactZipFile(hostURL, originAndArtifactPath, version)
if err == nil {
break
}
// TODO consider logging the missed fetch and or developing a bad reputation for the host.
}
if zipFileContents == nil {
err = fmt.Errorf("Search of Internet did not find relish software artifact '%s'", originAndArtifactPath)
return
}
artifactKnownToBeReplica = true
// Unzip the artifact into the proper local directory tree
// TODO TODO Really don't know the artifact version here in some case, (in case there was nothing
// not even a metadata.txt file locally, and no version was specified on command line) so
// we don't have the correct path for artifactVersionDir known yet in that case !!!
// WE DO KNOW IT HAS TO BE A SHARED REPLICASE DIR PATH however.
versionStr := "/v" + version
artifactVersionDir = ldr.RelishRuntimeLocation + "/shared/relish/replicas/" + originAndArtifactPath + versionStr
//gos.MkdirAll(name string, perm FileMode) error
var perm os.FileMode = 0777
err = gos.MkdirAll(artifactVersionDir, perm)
if err != nil {
return
}
zipFilePath := ldr.RelishRuntimeLocation + "/shared/relish/replicas/" + originAndArtifactPath + "/" + zipFileName
err = gos.WriteFile(zipFilePath, zipFileContents, perm)
if err != nil {
return
}
// Open an artifact version zip archive for reading.
var srcZipFileContents []byte
srcZipFileContents, err = zip_util.ExtractFileFromZipFileContents(zipFileContents, "artifactVersionContents.zip")
if err != nil {
return
}
/////////////////////////////////////////////////////////////////////////////////////
// Verify the signed contents using digital signature verification.
var sharedRelishPublicKeyCertificateBytes []byte
var sharedRelishPublicKeyCertificate string
var installationSharedRelishPublicKeyCert string
var originPublicKeyCertificateBytes []byte
var originPublicKeyCertificate string
var signatureBytes []byte
var signature string
sharedRelishPublicKeyCertificateBytes, err = zip_util.ExtractFileFromZipFileContents(zipFileContents, "sharedRelishPublicKeyCertificate.pem")
if err != nil {
return
}
sharedRelishPublicKeyCertificate = strings.TrimSpace(string(sharedRelishPublicKeyCertificateBytes))
originPublicKeyCertificateBytes, err = zip_util.ExtractFileFromZipFileContents(zipFileContents, "originPublicKeyCertificate.pem")
if err != nil {
return
}
originPublicKeyCertificate = strings.TrimSpace(string(originPublicKeyCertificateBytes))
signatureBytes, err = zip_util.ExtractFileFromZipFileContents(zipFileContents, "signatureOfArtifactVersionContents.pem")
if err != nil {
return
}
signature = strings.TrimSpace(string(signatureBytes))
installationSharedRelishPublicKeyCert, err = crypto_util.GetPublicKeyCert("origin", "shared.relish.pl2012")
if err != nil {
// Could not find shared relish pl public key cert in keys/public directory of this relish project directory.
// Try downloading the public key from http://shared.relish.pl and installing it in the project directory.
var cert string
cert, err = FetchSharedRelishPublicKeyCert()
if err != nil {
return
}
err = crypto_util.StorePublicKeyCert("origin", "shared.relish.pl2012", cert)
if err != nil {
return
}
installationSharedRelishPublicKeyCert, err = crypto_util.GetPublicKeyCert("origin", "shared.relish.pl2012")
if err != nil {
return
}
}
installationSharedRelishPublicKeyCert = strings.TrimSpace(installationSharedRelishPublicKeyCert)
if err != nil {
return
}
// Is the shared relish public key cert in the artifact zip file identical to the cert that came with my
// relish distribution? If not, panic.
if sharedRelishPublicKeyCertificate != installationSharedRelishPublicKeyCert {
err = fmt.Errorf("Did not install downloaded artifact because shared.relish.pl2012 public key certificate\n"+
"in artifact %s (v%s) downloaded from %s\n"+
"is different than shared.relish.pl2012 public key certificate in this relish installation.\n",
originAndArtifactPath, version, hostURL)
return
}
// Validate that shared.relish.pl2012 public key is signed properly, obtaining the shared.relish.pl2012 publicKeyPEM.
sharedRelishPublicKey := crypto_util.VerifiedPublicKey("", sharedRelishPublicKeyCertificate, "origin", "shared.relish.pl2012")
if sharedRelishPublicKey == "" {
err = errors.New("Invalid shared.relish.pl2012 public key certificate.")
return
}
// Validate the artifact-publishing origin's public key cert
slashPos := strings.Index(originAndArtifactPath, "/")
originId := originAndArtifactPath[:slashPos]
originPublicKey := crypto_util.VerifiedPublicKey(sharedRelishPublicKey, originPublicKeyCertificate, "origin", originId)
if originPublicKey == "" {
err = fmt.Errorf("Did not install downloaded artifact because %s public key certificate\n"+
"in artifact %s (v%s) downloaded from %s\n"+
"is invalid.\n",
originId, originAndArtifactPath, version, hostURL)
return
}
signedContent := zipFileName + "_|_" + string(srcZipFileContents)
if !crypto_util.Verify(originPublicKey, signature, signedContent) {
err = fmt.Errorf("Did not install downloaded artifact because artifact version content\n"+
"in artifact %s (v%s) downloaded from %s\n"+
"does not match (was not verified by) its digital signature.\n",
originAndArtifactPath, version, hostURL)
return
}
// Woohoo! Contents are verified.
//
/////////////////////////////////////////////////////////////////////////////////////
//
// Write them to relish installation shared code directory tree.
// Note: Assuming the artifactVersionContents.zip file starts with src/ pkg/ doc/ etc not with v0002/
err = zip_util.ExtractZipFileContents(srcZipFileContents, artifactVersionDir)
if err != nil {
return
}
Log(ALWAYS_, "Downloaded %s (v%s) from %s\n", originAndArtifactPath, version, hostURL)
}
if !artifactAlreadyLoaded { // Read built.txt from the artifact version directory
builtFilePath := artifactVersionDir + "/built.txt"
var builtFileContents []byte
_, statErr := gos.Stat(builtFilePath)
if statErr == nil {
builtFileContents, err = gos.ReadFile(builtFilePath)
if err != nil {
return
}
artifactsVersionsStrs := strings.Fields(string(builtFileContents))
n := len(artifactsVersionsStrs)
for i := 0; i < n; i += 2 {
artifactPath := artifactsVersionsStrs[i]
artifactVersion := artifactsVersionsStrs[i+1]
alreadyDesiredVersion, versionFound := ldr.LoadedArtifacts[artifactPath]
if versionFound {
if artifactVersion != alreadyDesiredVersion {
Log(ALWAYS_, "Using v%s of %s. %s (v%s) may prefer v%s of %s.\n", alreadyDesiredVersion, artifactPath, originAndArtifactPath, version, artifactVersion, artifactPath)
}
} else {
// Tell the loader to prefer the version of the other artifact
// that the being-loaded artifact built.txt file specifies.
ldr.LoadedArtifacts[artifactPath] = artifactVersion
}
}
} else if !os.IsNotExist(statErr) {
err = fmt.Errorf("Can't stat '%s': %v\n", builtFilePath, statErr)
return
}
// else there is no built.txt file. Accept that for now
// and subsequently load the current versions of artifacts where no other version is preferred.
}
////////////////////////////////////////////////////////////////////////
// TODO
// Load all the code files in the package of the artifact.
// compile files if necessary?
//
// record the version of the artifact and package in the loader's registry of loaded packages and artifacts.
packageSourcePath := artifactVersionDir + "/src/" + packagePath
packageCompiledPath := artifactVersionDir + "/pkg/" + packagePath
// Read the filenames of source files etc. in the /src/... package directory
var sourceDirFile *os.File
sourceDirFile, err = gos.Open(packageSourcePath)
if err != nil {
return
}
defer sourceDirFile.Close()
var filenames []string
filenames, err = sourceDirFile.Readdirnames(-1)
if err != nil {
return
}
// Create /pkg/ dir tree if does not exist already
_, statErr := gos.Stat(packageCompiledPath)
if statErr != nil {
if !os.IsNotExist(statErr) {
err = fmt.Errorf("Can't stat relish intermediate-code directory '%s': %v\n", packageCompiledPath, statErr)
return
}
var perm os.FileMode = 0777
err = gos.MkdirAll(packageCompiledPath, perm)
if err != nil {
return
}
}
ldr.LoadedArtifacts[originAndArtifactPath] = version
ldr.LoadedArtifactKnownToBePublished[originAndArtifactPath] = strings.Contains(artifactVersionDir, "/shared/relish/artifacts/")
ldr.LoadedArtifactKnownToBeReplica[originAndArtifactPath] = artifactKnownToBeReplica
Log(LOAD2_, "ldr.LoadedArtifactKnownToBePublished[%s]=%v\n", originAndArtifactPath, ldr.LoadedArtifactKnownToBePublished[originAndArtifactPath])
Log(LOAD2_, "ldr.LoadedArtifactKnownToBeReplica[%s]=%v\n", originAndArtifactPath, ldr.LoadedArtifactKnownToBeReplica[originAndArtifactPath])
Logln(LOAD_, "artifactVersionDir="+artifactVersionDir)
ldr.LoadedArtifactKnownToBeLocal[originAndArtifactPath] = !(ldr.LoadedArtifactKnownToBePublished[originAndArtifactPath] || artifactKnownToBeReplica)
if relish.DatabaseURI() == "" {
dbDirPath := ldr.databaseDirPath(originAndArtifactPath)
var perm os.FileMode = 0777
err = gos.MkdirAll(dbDirPath, perm)
if err != nil {
return
}
dbFilePath := dbDirPath + "/" + ldr.DatabaseName
relish.SetDatabaseURI(dbFilePath) // TODO NOT TRUE AT ALL YET
// This can be overridden with a statement in the program, as long as a persistence op has not been used first.
}
// Collect a map of file nodes to the root of the filename. We will be passing this to the generator to generate runtime
// code for the whole package all at once.
//
astFileNodes := make(map[*ast.File]string)
for _, filename := range filenames {
var sourceFound bool
var pickledFound bool
if strings.HasSuffix(filename, ".rel") { // consider only the relish source files in the dir.
// This is actually quite controversial, since it means that source code MUST be present
// or we won't bother looking for the compiled file to load.
// This is a somewhat political opinionated decision. Will have to be seriously mulled if not pondered.
// TODO add in here the on-demand compilation as found in relish.go
// Doing it NOW!!
sourceFilePath := packageSourcePath + "/" + filename
fileNameRoot := filename[:len(filename)-4]
pickleFilePath := packageCompiledPath + "/" + fileNameRoot + ".rlc"
sourceFileInfo, statErr := gos.Stat(sourceFilePath)
if statErr != nil {
if !os.IsNotExist(statErr) {
err = fmt.Errorf("Can't stat relish source file '%s': %v\n", sourceFilePath, statErr)
return
}
} else {
sourceFound = true
}
pickleFileInfo, statErr := gos.Stat(pickleFilePath)
if statErr != nil {
if !os.IsNotExist(statErr) {
err = fmt.Errorf("Can't stat relish intermediate-code file '%s': %v\n", pickleFilePath, statErr)
return
}
} else {
pickledFound = true
}
var parseNeeded bool
if sourceFound {
if pickledFound {
if sourceFileInfo.ModTime().After(pickleFileInfo.ModTime()) {
parseNeeded = true
}
} else {
parseNeeded = true
}
} else if !pickledFound {
err = fmt.Errorf("Error: Found neither relish source file '%s' nor intermediate-code file '%s'.\n", sourceFilePath, pickleFilePath)
return
}
var fileNode *ast.File
if parseNeeded {
var fset = token.NewFileSet()
fileNode, err = parser.ParseFile(fset, sourceFilePath, nil, parserDebugMode)
if err != nil {
err = fmt.Errorf("Error parsing file '%s': %v\n", sourceFilePath, err)
return
}
err = ast.Pickle(fileNode, pickleFilePath)
if err != nil {
err = fmt.Errorf("Error pickling file '%s': %v\n", sourceFilePath, err)
return
}
} else { // read the pickled (intermediate-code) file
fileNode, err = ast.Unpickle(pickleFilePath)
if err != nil {
err = fmt.Errorf("Error unpickling file '%s': %v\n", pickleFilePath, err)
return
}
}
err = ldr.ensureImportsAreLoaded(fileNode)
if err != nil {
return
}
astFileNodes[fileNode] = fileNameRoot
// gen = generator.NewGenerator(fileNode, fileNameRoot) // TODO NOW add a isLocal =ldr.LoadedArtifactKnownToBeLocal[originAndArtifactPath]
// argument so that we can flag the RPackage object as local or shared.
// gen.GenerateCode()
if packageIdentifier != fileNode.Name.Name {
err = fmt.Errorf("\nThe origin, artifact, or package metadata at top of source code file\n'%s'\ndoes not match the package directory path where the file resides.\n", sourceFilePath)
return
}
if parseNeeded {
if !ldr.quiet {
Log(ALWAYS_, "Compiled %s\n", sourceFilePath)
}
}
} // end of if it is a code file.
} // end of loop over each file in the package.
if len(astFileNodes) > 0 {
gen = generator.NewGenerator(astFileNodes)
gen.GenerateCode()
}
native_methods.WrapNativeMethods(packageIdentifier) // Check if package has native methods, if so, make RMethod wrappers.
ldr.LoadedPackages[packageIdentifier] = version
delete(ldr.PackagesBeingLoaded, packageIdentifier)
if !ldr.quiet {
Log(ALWAYS_, "Loaded %s\n", packageCompiledPath)
}
return
}
/*
Remove the existing shared copy of the version of the artifact (in a failsafe manner).
Copy source code for the version of the package to the shared sourcecode tree.
Create a zip file of the version of the artifact.
Wrap that with another zip file which will eventually contain a certificate
of the origin's public key, and a signature of (the SHA256 hash of) the inner zip file contents.
Name the outer zip file by an unambiguous name corresponding to the version of the artifact from the origin.
Check to see if there is a metadata file in the shared directory for the artifact.
If no metadata file in shared, add it. Metadata file name should again be fully qualified unambiguous name,
to ensure findability by Google search. Or the content should contain a standard findable title like
"Relish Artifact Metadata".
Note that the metadata.txt file should eventually also include,
below its meaningful text content, a certificate of the origin's public key, and a signature of (the SHA256 hash of)
the meaningful content.
Should copy local artifact version directory tree that was published to create the next version to continue work on.
Note about signed code plans
============================
The plan is to have every copy of relish include the public key of shared.relish.pl.
This key can be directly compared manually at any time with the public key published at the site shared.relish.pl.
Each code-origin owner who wants to officially publish relish code should register their origin at shared.relish.pl
and should receive back both
1) a certificate (signed by shared.relish.pl and verifiable with shared.relish.pl's public key)
where such certificate attests to the association between another public key and the origin name.
2) The actual origin public key
3) A corresponding private key.
The origin owner should keep their private key secret but install it in a standard place
within their relish development environment so that their relish instance can use it to sign code.
The origin owner should also keep their certificate of their public key installed at a standard place
in their relish development environment, so they can include that cert in signed-code outer zip files.
The certified public key should actually be published at a standard place in the canonical server
for the origin, and also at shared.relish.pl. This is so that such a key can periodically be verified
as being a currently valid one.
Note: If the private key is stolen, someone else can produce code signed as coming from your origin,
so maybe the same password used to sign up to register your origin should serve as a decryption key
for a symmetrically encrypted version of the private key. So you would be prompted to enter the password
when publishing (and signing) some code.
If not using encrypted private key, then if it is discovered that the key has been stolen (e.g. some code
fraudulently claiming to be from the origin is discovered), then a solution would be to apply for a
new public key for the origin, and re-publish all legitimate code signed by the new key.
Periodically, a imported-code-using instance of relish should re-verify (at the canonical server or shared.relish.pl)
that the public key that is signing each of their imported artifacts is still the valid public key for the origin.
Perhaps shared.relish.pl can contain a timestamp of when the most recent re-keying incident happened,
and relish instances can periodically check that to see if re-verification of origin public-keys, and
possible redownloading of re-signed code artifacts is necessary.
*/
func PublishSourceCode(relishRoot string, originAndArtifact string, version string) (err error) {
slashPos := strings.Index(originAndArtifact, "/")
originId := originAndArtifact[:slashPos]
// Obtain the private key for the origin that is publishing.
originPrivateKey, err := crypto_util.GetPrivateKey("origin", originId)
if err != nil {
return
}
// Obtain the public key certificate for the origin that is publishing.
originPublicKeyCertificate, err := crypto_util.GetPublicKeyCert("origin", originId)
if err != nil {
return
}
// Obtain the public key certificate of shared.relish.pl2012
sharedRelishPublicKeyCertificate, err := crypto_util.GetPublicKeyCert("origin", "shared.relish.pl2012")
if err != nil {
return
}
// Validate that it is signed properly, obtaining the shared.relish.pl2012 publicKeyPEM.
sharedRelishPublicKey := crypto_util.VerifiedPublicKey("", sharedRelishPublicKeyCertificate, "origin", "shared.relish.pl2012")
if sharedRelishPublicKey == "" {
err = errors.New("Invalid shared.relish.pl2012 public key certificate.")
return
}
// Do a quick validation of publishing origin's public key cert
originPublicKey := crypto_util.VerifiedPublicKey(sharedRelishPublicKey, originPublicKeyCertificate, "origin", originId)
if originPublicKey == "" {
err = errors.New("Invalid " + originId + " public key certificate.")
return
}
// prompt for the publishing origin's private key password.
var buf *bufio.Reader = bufio.NewReader(os.Stdin)
fmt.Print("Enter code-origin administration password:"******"/artifacts/" + originAndArtifact + "/"
sharedArtifactPath := relishRoot + "/shared/relish/artifacts/" + originAndArtifact + "/"
// Check if metadata.txt file exists in shared. If not, create it by copying local metadata.txt file
sharedMetadataPath := sharedArtifactPath + "metadata.txt"
_, err = gos.Stat(sharedMetadataPath)
if err != nil {
if os.IsNotExist(err) {
err = gos.MkdirAll(sharedArtifactPath, 0777)
if err != nil {
fmt.Printf("Error making shared artifact directory %s: %s\n", sharedArtifactPath, err)
return
}
localMetadataPath := localArtifactPath + "metadata.txt"
var content []byte
content, err = gos.ReadFile(localMetadataPath)
if err != nil {
return
}
err = gos.WriteFile(sharedMetadataPath, content, 0666)
if err != nil {
return
}
} else {
fmt.Printf("Can't stat '%s' : %v\n", sharedArtifactPath+"metadata.txt", err)
return
}
}
// Note. This does not update the shared metadata.txt file from the local if the shared metadata.txt
// file already existed.
//
versionPath := "v" + version
sharedArtifactVersionPath := sharedArtifactPath + versionPath
_, err = gos.Stat(sharedArtifactVersionPath)
foundVersionShared := false
if err != nil {
if !os.IsNotExist(err) {
fmt.Printf("Can't stat directory '%s' : %v\n", sharedArtifactVersionPath, err)
return
}
} else {
foundVersionShared = true
}
if foundVersionShared {
fmt.Printf("%s already exists. Cannot republish the same version.\n", sharedArtifactVersionPath)
return
}
localSrcDirPath := localArtifactPath + versionPath + "/src"
srcDirPath := sharedArtifactVersionPath + "/src"
localDocDirPath := localArtifactPath + versionPath + "/doc"
docDirPath := sharedArtifactVersionPath + "/doc"
// mkdir the version of the shared directory
err = gos.MkdirAll(sharedArtifactVersionPath, 0777)
if err != nil {
fmt.Printf("Error making shared artifact version directory %s: %s\n", sharedArtifactVersionPath, err)
return
}
// Copy source code directory tree to "shared/relish/artifacts" tree root.
err = copySrcDirTree(localSrcDirPath, srcDirPath)
if err != nil {
fmt.Printf("Error copying local src dir to create %s: %s\n", srcDirPath, err)
return
}
_, statErr := gos.Stat(localDocDirPath)
if statErr == nil {
err = copyDocDirTree(localDocDirPath, docDirPath)
if err != nil {
fmt.Printf("Error copying local doc dir to create %s: %s\n", docDirPath, err)
return
}
}
// TBD
// Zip the source and docs!
srcZipFilePath := sharedArtifactVersionPath + "/artifactVersionContents.zip"
err = zipSrcAndDocDirTrees(srcDirPath, docDirPath, srcZipFilePath)
if err != nil {
fmt.Printf("Error zipping %s and %s: %s\n", srcDirPath, docDirPath, err)
return
}
// Now have to sign it and put into an outer zip file.
err = signZippedSrc(srcZipFilePath, originPrivateKey, originPrivateKeyPassword, originPublicKeyCertificate, sharedRelishPublicKeyCertificate, sharedArtifactPath, originAndArtifact, version)
if err != nil {
fmt.Printf("Error signing %s: %s\n", srcZipFilePath, err)
return
}
err = gos.Remove(srcZipFilePath)
if err != nil {
fmt.Printf("Error removing %s: %s\n", srcZipFilePath, err)
return
}
return
}