/*
Serializes a relish abstract syntax tree into a file. This file can be thought of as an intermediate code file,
since it contains the pre-parsed tree.
*/
func Pickle(fileNode *File, pickleFilePath string) (err error) {
var file *os.File
file, err = gos.Create(pickleFilePath)
defer file.Close()
encoder := gob.NewEncoder(file)
err = encoder.Encode(fileNode)
return
}
/*
Zips the specified directory tree of relish source code files into the specified zip file.
*/
func zipSrcAndDocDirTrees(srcDirectoryPath string, docDirectoryPath string, zipFilePath string) (err error) {
var buf *bytes.Buffer
buf, err = zipSrcDirTree1(srcDirectoryPath, docDirectoryPath)
var file *os.File
file, err = gos.Create(zipFilePath)
if err != nil {
return
}
_, err = buf.WriteTo(file)
if err != nil {
return
}
err = file.Close()
return
}
/*
Given a zip file of the source code directory tree,
1. Computes the SHA256 hash of the source code zip file contents, then signs the hash using
the private key of the origin.
2. Adds
a. the certificate of the origin's public key (including that public key), and
b. the signature of the source zip file (which can be verified with that public key)
c. the source zip file
to an outer (wrapper) zip file that it is creating.
3. Writes the wrapper zip file as e.g. a.b.com2013--my_artifact_name--1.0.3.zip to the
shared artifact's root directory.
NOTE: STEPS 1 and 2. a. b. are TBD !!!! Just re-zips the src.zip file presently.
*/
func signZippedSrc(srcZipPath string,
originPrivateKey string,
originPrivateKeyPassword string,
originPublicKeyCertificate string,
sharedRelishPublicKeyCertificate string,
sharedArtifactPath string,
originAndArtifact string,
version string) (err error) {
originAndArtifactFilenamePart := strings.Replace(originAndArtifact, "/", "--", -1)
wrapperFilename := originAndArtifactFilenamePart + "---" + version + ".zip"
wrapperFilePath := sharedArtifactPath + "/" + wrapperFilename
var srcZipContents []byte
srcZipContents, err = gos.ReadFile(srcZipPath)
if err != nil {
return
}
content := wrapperFilename + "_|_" + string(srcZipContents)
signaturePEM, err := crypto_util.Sign(originPrivateKey, originPrivateKeyPassword, content)
var buf *bytes.Buffer
buf, err = signZippedSrc1(srcZipPath, originPublicKeyCertificate, sharedRelishPublicKeyCertificate, signaturePEM)
var file *os.File
file, err = gos.Create(wrapperFilePath)
if err != nil {
return
}
_, err = buf.WriteTo(file)
if err != nil {
return
}
err = file.Close()
return
}
/*
Extracts contents of a zip file into the specified directory, which must exist and be writeable.
The directory path must not end with a "/".
The directory path must be expressed in unix-style path syntax, with "/" separators not "\" or "\\"
Always excludes __MACOSX folders from what it writes to the target directory tree.
*/
func ExtractZipFileContents(zipFileContents []byte, dirPath string) (err error) {
var perm os.FileMode = 0777
byteSliceReader := bytes.NewReader(zipFileContents)
var r *zip.Reader
r, err = zip.NewReader(byteSliceReader, int64(len(zipFileContents)))
if err != nil {
return
}
// Iterate through the files in the archive,
// copying their contents.
for _, f := range r.File {
fileInfo := f.FileHeader.FileInfo()
// if strings.Index(fileInfo.Name(),"__MACOSX") == 0 {
// replaced with following line to be Go1.2 compatible.
if strings.Index(f.FileHeader.Name, "__MACOSX") == 0 {
continue
}
if fileInfo.IsDir() {
Log(LOAD2_, "Directory %s:\n", f.Name)
err = gos.MkdirAll(dirPath+"/"+f.Name, perm)
if err != nil {
return
}
} else {
Log(LOAD2_, "Copying file %s:\n", f.Name)
var rc io.ReadCloser
rc, err = f.Open()
if err != nil {
return
}
slashPos := strings.LastIndex(f.Name, "/")
if slashPos != -1 {
relativeDirPath := f.Name[:slashPos]
err = gos.MkdirAll(dirPath+"/"+relativeDirPath, perm)
if err != nil {
return
}
}
var outFile *os.File
outFile, err = gos.Create(dirPath + "/" + f.Name)
if err != nil {
return
}
_, err = io.Copy(outFile, rc)
if err != nil {
return
}
rc.Close()
outFile.Close()
Logln(LOAD2_)
}
}
return
}
func main() {
var loggingLevel int
var webListeningPort int
var tlsWebListeningPort int
var shareListeningPort int // port on which source code will be shared by http
var sharedCodeOnly bool // do not use local artifacts - only those in shared directory.
var explorerListeningPort int // port on which data explorer_api web service will be served.
var runningArtifactMustBeFromShared bool
var dbName string
var cpuprofile string
var publish bool
var quiet bool
var projectPath string
// var gcIntervalSeconds int
//var fset = token.NewFileSet()
flag.IntVar(&loggingLevel, "log", 0, "The logging level: 0 is least verbose, 2 most")
flag.IntVar(&webListeningPort, "web", 0, "The http listening port - if not supplied, does not listen for http requests")
flag.IntVar(&tlsWebListeningPort, "tls", 0, "The https listening port - if not supplied, does not listen for https requests")
flag.IntVar(&explorerListeningPort, "explore", 0, "The explorer_api web service listening port - if supplied, the data explorer tool can connect to this program on this port")
flag.StringVar(&dbName, "db", "db1", "The database name. A SQLITE database file called <name>.db will be created/used in artifact data directory")
flag.BoolVar(&sharedCodeOnly, "shared", false, "Use shared version of all artifacts - ignore local/dev copy of artifacts")
flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")
flag.IntVar(&shareListeningPort, "share", 0, "The code sharing http listening port - if not supplied, does not listen for source code sharing http requests")
flag.BoolVar(&publish, "publish", false, "artifactpath version - copy specified version of artifact to shared/relish/artifacts")
flag.BoolVar(&quiet, "quiet", false, "do not show package loading info or interpreter version in program output")
flag.StringVar(&projectPath, "init", "", "<artifactpath> [webapp] - create directory tree and template files for a relish software project")
flag.IntVar(¶ms.GcIntervalSeconds, "gc", params.GcIntervalSeconds, "The garbage collection check interval (seconds): defaults to 20")
flag.IntVar(¶ms.DbMaxConnections, "pool", params.DbMaxConnections, "Maximum number of db connections to open with the database: defaults to 1")
flag.IntVar(¶ms.DbMaxReadConnections, "rpool", params.DbMaxReadConnections, "Maximum number of read-only db connections to open with the database")
flag.IntVar(¶ms.DbMaxWriteConnections, "wpool", params.DbMaxWriteConnections, "Maximum number of writeable db connections to open with the database")
flag.Parse()
pathParts := flag.Args() // full path to package, or originAndArtifact and path to package
// (or originAndArtifact and version number if -publish)
if cpuprofile != "" {
f, err := gos.Create(cpuprofile)
if err != nil {
fmt.Println(err)
return
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
dbg.InitLogging(int32(loggingLevel))
//relish.InitRuntime("relish.db")
// if ! publish {
// builtin.InitBuiltinFunctions()
// }
var g *generator.Generator
var relishRoot string // This actually has to be the root of the runtime environment
// i.e. /opt/relish if this is a binary distribution,
// or /opt/relish/rt if this is a source distribution
workingDirectory, err := gos.Getwd()
if err != nil {
fmt.Printf("Cannot determine working directory: %s\n", err)
}
var originAndArtifact string
var version string
var packagePath string
isSubDir := false
isSourceDist := false
relishIndexInWd := strings.Index(workingDirectory, "/relish/")
if relishIndexInWd > -1 {
isSubDir = true
} else {
relishIndexInWd = strings.Index(workingDirectory, "/relish")
if relishIndexInWd == -1 || relishIndexInWd != len(workingDirectory)-7 {
if projectPath != "" { // Must be creating a relish dir under a new project dir
relishRoot = workingDirectory + "/relish"
err = gos.MkdirAll(relishRoot, 0777)
if err != nil {
fmt.Printf("Error making relish project directory %s: %s\n", relishRoot, err)
return
}
} else {
fmt.Printf("relish command must be run from within a relish directory tree.\n")
return
}
}
}
if relishRoot == "" {
relishRoot = workingDirectory[:relishIndexInWd+7]
}
_, err = gos.Stat(relishRoot + "/rt")
if err == nil {
relishRoot += "/rt"
isSourceDist = true
} else if !os.IsNotExist(err) {
fmt.Printf("Can't stat '%s' : %v\n", relishRoot+"/rt", err)
return
}
// relishRoot is now established
if projectPath != "" {
if len(pathParts) < 1 {
err = initProject(relishRoot, projectPath, "")
if err == nil {
fmt.Printf("\nCreated relish project template\n%s/artifacts/%s\n", relishRoot, projectPath)
fmt.Printf("\nTo run your project template's dummy main program, relish %s\n\n", projectPath)
} else {
fmt.Printf("\nError initializing project %s: %s\n", projectPath, err)
}
} else {
projectType := pathParts[0]
err = initProject(relishRoot, projectPath, projectType)
if err == nil {
fmt.Printf("\nCreated relish web-app project template\n%s/artifacts/%s\n", relishRoot, projectPath)
fmt.Printf("\nTo run the web-app, relish -web 8080 %s\n", projectPath)
fmt.Printf("Then enter localhost:8080 into your browser's address bar to view the web app.\n\n")
} else {
fmt.Printf("\nError initializing project %s: %s\n", projectPath, err)
}
}
return
}
if isSubDir { // See where we are more specifically
if isSourceDist {
idx := strings.Index(workingDirectory, "/rt/shared")
if idx > -1 {
runningArtifactMustBeFromShared = true
}
} else {
idx := strings.Index(workingDirectory, "/relish/shared")
if idx > -1 {
runningArtifactMustBeFromShared = true
}
}
if !publish {
// See if the current directory is a particular artifact version directory,
// or even is a particular package directory.
originPos := strings.Index(workingDirectory, "/artifacts/") + 11
if originPos == 10 {
originPos = strings.Index(workingDirectory, "/replicas/") + 10
}
if originPos >= 10 {
match := reVersionedPackage.FindStringSubmatchIndex(workingDirectory)
if match != nil {
version = workingDirectory[match[2]:match[3]]
originAndArtifact = workingDirectory[originPos:match[0]]
packagePath = workingDirectory[match[3]+1:]
} else {
match := reVersionAtEnd.FindStringSubmatch(workingDirectory)
if match != nil {
version = match[1]
originAndArtifact = workingDirectory[originPos : len(workingDirectory)-len(version)-2]
}
}
}
}
}
if !publish {
builtin.InitBuiltinFunctions(relishRoot)
}
crypto_util.SetRelishRuntimeLocation(relishRoot) // So that keys can be fetched.
if publish {
if len(pathParts) < 2 {
fmt.Println("Usage (example): relish -publish someorigin.com2013/artifact_name 1.0.23")
return
}
originAndArtifact = pathParts[0]
version = pathParts[1]
if strings.HasSuffix(originAndArtifact, "/") { // Strip trailing / if present
originAndArtifact = originAndArtifact[:len(originAndArtifact)-1]
}
err = global_publisher.PublishSourceCode(relishRoot, originAndArtifact, version)
if err != nil {
fmt.Println(err)
}
return
}
sourceCodeShareDir := ""
if shareListeningPort != 0 {
// sourceCodeShareDir hould be the "relish/shared"
// or "relish/rt/shared" of "relish/4production/shared" or "relish/rt/4production/shared" directory.
sourceCodeShareDir = relishRoot + "/shared"
}
onlyCodeSharing := (shareListeningPort != 0 && webListeningPort == 0 && tlsWebListeningPort == 0)
if onlyCodeSharing {
if shareListeningPort < 1024 && shareListeningPort != 80 {
fmt.Println("Error: The source-code sharing port must be 80 or > 1023 (8421 is the standard if using a high port)")
return
}
web.ListenAndServeSourceCode(shareListeningPort, sourceCodeShareDir)
// The previous call should wait on incoming socket connections on the specified port, and never return.
// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
// have permission to listen on a < 1024 port like port 80.
fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
"Did you setup to give relish permission to bind to privileged ports?\n"+
"or is another process already listening on this port?\n", shareListeningPort)
return
}
var loader = global_loader.NewLoader(relishRoot, sharedCodeOnly, dbName+".db", quiet)
if originAndArtifact == "" {
if len(pathParts) == 3 { // originAndArtifact version packagePath
originAndArtifact = pathParts[0]
version = pathParts[1]
packagePath = pathParts[2]
} else if len(pathParts) == 2 { // originAndArtifact packagePath
originAndArtifact = pathParts[0]
packagePathOrVersion := pathParts[1]
// Determine if is a version by regexp or a package path has been supplied
version = reVersion.FindString(packagePathOrVersion)
if version == "" {
packagePath = packagePathOrVersion
}
} else if shareListeningPort == 0 || webListeningPort != 0 || tlsWebListeningPort != 0 {
if len(pathParts) != 1 {
fmt.Println("Usage: relish [-web 80] originAndArtifact [version] [path/to/package]\n# package path defaults to main")
return
}
originAndArtifact = pathParts[0]
}
} else if packagePath == "" {
if len(pathParts) == 1 {
packagePath = pathParts[0]
} else if len(pathParts) > 1 {
fmt.Println("Usage (when in an artifact version directory): relish [-web 80] [path/to/package]\n# package path defaults to main")
return
}
} else { // both originAndArtifact and packagePath are defined (non "")
if len(pathParts) != 0 {
fmt.Println("Usage (when in a package directory): relish [-web 80]")
return
}
}
if strings.HasSuffix(originAndArtifact, "/") { // Strip trailing / if present
originAndArtifact = originAndArtifact[:len(originAndArtifact)-1]
}
if strings.HasSuffix(packagePath, "/") { // Strip trailing / if present
packagePath = packagePath[:len(packagePath)-1]
}
if packagePath == "" {
packagePath = "main" // substitute a default.
}
fullPackagePath := fmt.Sprintf("%s/v%s/pkg/%s", originAndArtifact, version, packagePath)
fullUnversionedPackagePath := fmt.Sprintf("%s/pkg/%s", originAndArtifact, packagePath)
g, err = loader.LoadPackage(originAndArtifact, version, packagePath, runningArtifactMustBeFromShared)
if err != nil {
if version == "" {
fmt.Printf("Error loading package %s from current version of %s: %v\n", packagePath, originAndArtifact, err)
} else {
fmt.Printf("Error loading package %s: %v\n", fullPackagePath, err)
}
return
}
g.Interp.SetRunningArtifact(originAndArtifact)
g.Interp.SetPackageLoader(loader)
// TODO the following rather twisty logic (from here to end of main method) could be straightened out.
// One of its purposes is to ensure that the last http listener is run in this goroutine rather
// than in a background one. And there can be different numbers of listeners...
// Count how many separate listeners there will be.
numListeners := 0
numListening := 0 // how many of those are already listening?
if webListeningPort != 0 {
numListeners += 1
if shareListeningPort != 0 && shareListeningPort != webListeningPort {
numListeners += 1
}
}
if tlsWebListeningPort != 0 {
numListeners += 1
}
if explorerListeningPort != 0 {
numListeners += 1
}
// end counting listeners
// check for disallowed port numbers, and if not, load the packages needed for web app serving
if webListeningPort != 0 {
if webListeningPort < 1024 && webListeningPort != 80 {
fmt.Println("Error: The web listening port must be 80 or > 1023")
return
}
if shareListeningPort != webListeningPort && shareListeningPort != 0 && shareListeningPort < 1024 && shareListeningPort != 80 {
fmt.Println("Error: The source-code sharing port must be 80 or > 1023 (8421 is the standard if using a high port)")
return
}
}
if tlsWebListeningPort != 0 {
if tlsWebListeningPort < 1024 && tlsWebListeningPort != 443 {
fmt.Println("Error: The tls web listening port must be 443 or > 1023")
return
}
}
if webListeningPort != 0 || tlsWebListeningPort != 0 {
err = loader.LoadWebPackages(originAndArtifact, version, runningArtifactMustBeFromShared)
if err != nil {
if version == "" {
fmt.Printf("Error loading web packages from current version of %s: %v\n", originAndArtifact, err)
} else {
fmt.Printf("Error loading web packages from version %s of %s: %v\n", version, originAndArtifact, err)
}
return
}
}
// check for disallowed port numbers, and if not, load the package needed for explorer_api web service serving
if explorerListeningPort != 0 {
if explorerListeningPort < 1024 && explorerListeningPort != 80 {
fmt.Println("Error: The explorer listening port must be 80 or > 1023")
return
}
explorerApiOriginAndArtifact := "shared.relish.pl2012/explorer_api"
explorerApiPackagePath := "web"
_, err = loader.LoadPackage(explorerApiOriginAndArtifact, "", explorerApiPackagePath, false)
if err != nil {
fmt.Printf("Error loading package %s from current version of %s: %v\n",
explorerApiPackagePath,
explorerApiOriginAndArtifact,
err)
return
}
}
if numListeners > 0 { // If we'll be listening for http requests, run main in a background goroutine.
go g.Interp.RunMain(fullUnversionedPackagePath, quiet)
}
if webListeningPort != 0 || tlsWebListeningPort != 0 {
web.SetWebPackageSrcDirPath(loader.PackageSrcDirPath(originAndArtifact + "/pkg/web"))
if webListeningPort != 0 {
if shareListeningPort == webListeningPort {
numListening += 1
if numListening == numListeners {
web.ListenAndServe(webListeningPort, sourceCodeShareDir)
// The previous call should wait on incoming socket connections on the specified port, and never return.
// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
// have permission to listen on a < 1024 port like port 80.
fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
"Did you setup to give relish permission to bind to privileged ports?\n"+
"or is another process already listening on this port?\n", webListeningPort)
return
} else {
go web.ListenAndServe(webListeningPort, sourceCodeShareDir)
}
} else {
if shareListeningPort != 0 {
numListening += 1
go web.ListenAndServeSourceCode(shareListeningPort, sourceCodeShareDir)
}
numListening += 1
if numListening == numListeners {
web.ListenAndServe(webListeningPort, "")
// The previous call should wait on incoming socket connections on the specified port, and never return.
// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
// have permission to listen on a < 1024 port like port 80.
fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
"Did you setup to give relish permission to bind to privileged ports?\n"+
"or is another process already listening on this port?\n", webListeningPort)
return
} else {
go web.ListenAndServe(webListeningPort, "")
}
}
}
if tlsWebListeningPort != 0 {
tlsCertPath, tlsKeyPath, err := crypto_util.GetTLSwebServerCertAndKeyFilePaths()
if err != nil {
fmt.Printf("Error starting TLS web listener: %s\n", err)
return
}
numListening += 1
if numListening == numListeners {
web.ListenAndServeTLS(tlsWebListeningPort, tlsCertPath, tlsKeyPath)
// The previous call should wait on incoming socket connections on the specified port, and never return.
// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
// have permission to listen on a < 1024 port like port 80.
fmt.Printf("Error: Could not bind to port %d to listen for https connections.\n"+
"Did you setup to give relish permission to bind to privileged ports?\n"+
"or is another process already listening on this port?\n", tlsWebListeningPort)
return
} else {
go web.ListenAndServeTLS(tlsWebListeningPort, tlsCertPath, tlsKeyPath)
}
}
}
if explorerListeningPort != 0 {
web.ListenAndServeExplorerApi(explorerListeningPort)
// The previous call should wait on incoming socket connections on the specified port, and never return.
// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
// have permission to listen on a < 1024 port like port 80.
fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
"Did you setup to give relish permission to bind to privileged ports?\n"+
"or is another process already listening on this port?\n", explorerListeningPort)
return
}
// This will only be reached if numListeners == 0 or there is an error starting listeners.
// but don't want to re-run main if there was an error starting listeners.
if numListeners == 0 {
g.Interp.RunMain(fullUnversionedPackagePath, quiet)
}
}
/*
Run the garbage collector loop.
Checks every params.GcIntervalSeconds to see if it should run the GC.
Runs it if the current allocated-and-not-yet-freed memory is greater than twice
the lowest memory allocated-and-not-freed level since GC was last run.
!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!
TODO : !!!!!!!!! : Need to add a params.GcForcedIntervalSeconds e.g. 3 minutes worth
where it will definitely GC if that amount of time has elapsed.
*/
func (i *Interpreter) GCLoop() {
defer Un(Trace(GC2_, "GCLoop"))
var prevA uint64
var prevGcTime time.Time = time.Now()
var currentGcTime time.Time
var forced bool
var nGCs int64
for {
time.Sleep(time.Duration(params.GcIntervalSeconds) * time.Second)
// time.Sleep(4 * time.Second)
// See if too much time has passed since last relish GC
forced = false
currentGcTime = time.Now()
if currentGcTime.Sub(prevGcTime) > time.Duration(params.GcForceIntervalSeconds)*time.Second {
forced = true
}
runtime.ReadMemStats(&m)
if forced || (m.Alloc > prevA*2) || (m.Alloc > prevA+10000000) { // if grew double or by more than 10 MB approx
if forced {
Logln(GC_, "GC because more than", params.GcForceIntervalSeconds, "seconds since last GC. Prev Alloc", prevA, ", Alloc", m.Alloc)
} else {
Logln(GC_, "GC because Prev Alloc", prevA, ", Alloc", m.Alloc)
}
i.GC()
prevGcTime = currentGcTime
nGCs++
runtime.ReadMemStats(&m)
prevA = m.Alloc
Logln(GC_, "Sys", m.Sys, "Mallocs", m.Mallocs, "Frees", m.Frees)
Logln(GC_, "HeapAlloc", m.HeapAlloc, "HeapInuse", m.HeapInuse, "HeapIdle", m.HeapIdle, "HeapReleased", m.HeapReleased, "HeapObjects", m.HeapObjects, "HeapSys", m.HeapSys)
Logln(GC_, "StackInuse", m.StackInuse, "StackSys", m.StackSys)
if Logging(GC_) {
i.rt.DebugAttributesMemory()
}
if Logging(GC2_) {
memProfileFilename := fmt.Sprintf("memory%d.prof", nGCs)
f, err := gos.Create(memProfileFilename)
if err == nil {
pprof.WriteHeapProfile(f)
f.Close()
} else {
Logln(GC2_, "Unable to create memory profile file", memProfileFilename)
}
}
} else if m.Alloc < prevA {
prevA = m.Alloc
}
}
}