Пример #1
0
/*
   Serializes a relish abstract syntax tree into a file. This file can be thought of as an intermediate code file,
   since it contains the pre-parsed tree.
*/
func Pickle(fileNode *File, pickleFilePath string) (err error) {
	var file *os.File
	file, err = gos.Create(pickleFilePath)
	defer file.Close()

	encoder := gob.NewEncoder(file)

	err = encoder.Encode(fileNode)

	return
}
/*
Zips the specified directory tree of relish source code files into the specified zip file.
*/
func zipSrcAndDocDirTrees(srcDirectoryPath string, docDirectoryPath string, zipFilePath string) (err error) {

	var buf *bytes.Buffer
	buf, err = zipSrcDirTree1(srcDirectoryPath, docDirectoryPath)

	var file *os.File
	file, err = gos.Create(zipFilePath)
	if err != nil {
		return
	}

	_, err = buf.WriteTo(file)
	if err != nil {
		return
	}
	err = file.Close()

	return
}
/*
  Given a zip file of the source code directory tree,
  1. Computes the SHA256 hash of the source code zip file contents, then signs the hash using
  the private key of the origin.
  2. Adds
     a. the certificate of the origin's public key (including that public key), and
     b. the signature of the source zip file (which can be verified with that public key)
     c. the source zip file
     to an outer (wrapper) zip file that it is creating.
  3. Writes the wrapper zip file as e.g. a.b.com2013--my_artifact_name--1.0.3.zip to the
     shared artifact's root directory.

  NOTE: STEPS 1 and 2. a. b. are TBD !!!! Just re-zips the src.zip file presently.
*/
func signZippedSrc(srcZipPath string,
	originPrivateKey string,
	originPrivateKeyPassword string,
	originPublicKeyCertificate string,
	sharedRelishPublicKeyCertificate string,
	sharedArtifactPath string,
	originAndArtifact string,
	version string) (err error) {
	originAndArtifactFilenamePart := strings.Replace(originAndArtifact, "/", "--", -1)
	wrapperFilename := originAndArtifactFilenamePart + "---" + version + ".zip"
	wrapperFilePath := sharedArtifactPath + "/" + wrapperFilename

	var srcZipContents []byte
	srcZipContents, err = gos.ReadFile(srcZipPath)
	if err != nil {
		return
	}

	content := wrapperFilename + "_|_" + string(srcZipContents)
	signaturePEM, err := crypto_util.Sign(originPrivateKey, originPrivateKeyPassword, content)

	var buf *bytes.Buffer
	buf, err = signZippedSrc1(srcZipPath, originPublicKeyCertificate, sharedRelishPublicKeyCertificate, signaturePEM)

	var file *os.File
	file, err = gos.Create(wrapperFilePath)
	if err != nil {
		return
	}

	_, err = buf.WriteTo(file)
	if err != nil {
		return
	}
	err = file.Close()

	return
}
Пример #4
0
/*
Extracts contents of a zip file into the specified directory, which must exist and be writeable.
The directory path must not end with a "/".
The directory path must be expressed in unix-style path syntax, with "/" separators not "\" or "\\"
Always excludes __MACOSX folders from what it writes to the target directory tree.
*/
func ExtractZipFileContents(zipFileContents []byte, dirPath string) (err error) {

	var perm os.FileMode = 0777

	byteSliceReader := bytes.NewReader(zipFileContents)

	var r *zip.Reader
	r, err = zip.NewReader(byteSliceReader, int64(len(zipFileContents)))
	if err != nil {
		return
	}

	// Iterate through the files in the archive,
	// copying their contents.
	for _, f := range r.File {

		fileInfo := f.FileHeader.FileInfo()
		// if strings.Index(fileInfo.Name(),"__MACOSX") == 0 {
		// replaced with following line to be Go1.2 compatible.
		if strings.Index(f.FileHeader.Name, "__MACOSX") == 0 {

			continue
		}
		if fileInfo.IsDir() {
			Log(LOAD2_, "Directory %s:\n", f.Name)
			err = gos.MkdirAll(dirPath+"/"+f.Name, perm)
			if err != nil {
				return
			}
		} else {
			Log(LOAD2_, "Copying file %s:\n", f.Name)
			var rc io.ReadCloser
			rc, err = f.Open()
			if err != nil {
				return
			}

			slashPos := strings.LastIndex(f.Name, "/")
			if slashPos != -1 {
				relativeDirPath := f.Name[:slashPos]
				err = gos.MkdirAll(dirPath+"/"+relativeDirPath, perm)
				if err != nil {
					return
				}
			}

			var outFile *os.File
			outFile, err = gos.Create(dirPath + "/" + f.Name)
			if err != nil {
				return
			}

			_, err = io.Copy(outFile, rc)
			if err != nil {
				return
			}
			rc.Close()
			outFile.Close()
			Logln(LOAD2_)
		}
	}
	return
}
Пример #5
0
func main() {
	var loggingLevel int
	var webListeningPort int
	var tlsWebListeningPort int
	var shareListeningPort int    // port on which source code will be shared by http
	var sharedCodeOnly bool       // do not use local artifacts - only those in shared directory.
	var explorerListeningPort int // port on which data explorer_api web service will be served.
	var runningArtifactMustBeFromShared bool
	var dbName string
	var cpuprofile string
	var publish bool
	var quiet bool
	var projectPath string
	// var gcIntervalSeconds int

	//var fset = token.NewFileSet()
	flag.IntVar(&loggingLevel, "log", 0, "The logging level: 0 is least verbose, 2 most")
	flag.IntVar(&webListeningPort, "web", 0, "The http listening port - if not supplied, does not listen for http requests")
	flag.IntVar(&tlsWebListeningPort, "tls", 0, "The https listening port - if not supplied, does not listen for https requests")

	flag.IntVar(&explorerListeningPort, "explore", 0, "The explorer_api web service listening port - if supplied, the data explorer tool can connect to this program on this port")
	flag.StringVar(&dbName, "db", "db1", "The database name. A SQLITE database file called <name>.db will be created/used in artifact data directory")

	flag.BoolVar(&sharedCodeOnly, "shared", false, "Use shared version of all artifacts - ignore local/dev copy of artifacts")

	flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")

	flag.IntVar(&shareListeningPort, "share", 0, "The code sharing http listening port - if not supplied, does not listen for source code sharing http requests")

	flag.BoolVar(&publish, "publish", false, "artifactpath version - copy specified version of artifact to shared/relish/artifacts")

	flag.BoolVar(&quiet, "quiet", false, "do not show package loading info or interpreter version in program output")

	flag.StringVar(&projectPath, "init", "", "<artifactpath> [webapp] - create directory tree and template files for a relish software project")

	flag.IntVar(&params.GcIntervalSeconds, "gc", params.GcIntervalSeconds, "The garbage collection check interval (seconds): defaults to 20")

	flag.IntVar(&params.DbMaxConnections, "pool", params.DbMaxConnections, "Maximum number of db connections to open with the database: defaults to 1")

	flag.IntVar(&params.DbMaxReadConnections, "rpool", params.DbMaxReadConnections, "Maximum number of read-only db connections to open with the database")

	flag.IntVar(&params.DbMaxWriteConnections, "wpool", params.DbMaxWriteConnections, "Maximum number of writeable db connections to open with the database")

	flag.Parse()

	pathParts := flag.Args() // full path to package, or originAndArtifact and path to package
	// (or originAndArtifact and version number if -publish)

	if cpuprofile != "" {
		f, err := gos.Create(cpuprofile)
		if err != nil {
			fmt.Println(err)
			return
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	dbg.InitLogging(int32(loggingLevel))
	//relish.InitRuntime("relish.db")

	//  if ! publish {
	//    	builtin.InitBuiltinFunctions()
	//	}

	var g *generator.Generator

	var relishRoot string // This actually has to be the root of the runtime environment
	// i.e. /opt/relish if this is a binary distribution,
	// or /opt/relish/rt if this is a source distribution

	workingDirectory, err := gos.Getwd()
	if err != nil {
		fmt.Printf("Cannot determine working directory: %s\n", err)
	}

	var originAndArtifact string
	var version string
	var packagePath string
	isSubDir := false
	isSourceDist := false

	relishIndexInWd := strings.Index(workingDirectory, "/relish/")
	if relishIndexInWd > -1 {
		isSubDir = true
	} else {
		relishIndexInWd = strings.Index(workingDirectory, "/relish")
		if relishIndexInWd == -1 || relishIndexInWd != len(workingDirectory)-7 {
			if projectPath != "" { // Must be creating a relish dir under a new project dir
				relishRoot = workingDirectory + "/relish"

				err = gos.MkdirAll(relishRoot, 0777)
				if err != nil {
					fmt.Printf("Error making relish project directory %s: %s\n", relishRoot, err)
					return
				}
			} else {
				fmt.Printf("relish command must be run from within a relish directory tree.\n")
				return
			}
		}
	}
	if relishRoot == "" {
		relishRoot = workingDirectory[:relishIndexInWd+7]
	}

	_, err = gos.Stat(relishRoot + "/rt")
	if err == nil {
		relishRoot += "/rt"
		isSourceDist = true
	} else if !os.IsNotExist(err) {
		fmt.Printf("Can't stat '%s' : %v\n", relishRoot+"/rt", err)
		return
	}

	// relishRoot is now established

	if projectPath != "" {
		if len(pathParts) < 1 {
			err = initProject(relishRoot, projectPath, "")
			if err == nil {
				fmt.Printf("\nCreated relish project template\n%s/artifacts/%s\n", relishRoot, projectPath)
				fmt.Printf("\nTo run your project template's dummy main program, relish %s\n\n", projectPath)
			} else {
				fmt.Printf("\nError initializing project %s: %s\n", projectPath, err)
			}
		} else {
			projectType := pathParts[0]
			err = initProject(relishRoot, projectPath, projectType)
			if err == nil {
				fmt.Printf("\nCreated relish web-app project template\n%s/artifacts/%s\n", relishRoot, projectPath)
				fmt.Printf("\nTo run the web-app, relish -web 8080 %s\n", projectPath)
				fmt.Printf("Then enter localhost:8080 into your browser's address bar to view the web app.\n\n")
			} else {
				fmt.Printf("\nError initializing project %s: %s\n", projectPath, err)
			}
		}
		return
	}

	if isSubDir { // See where we are more specifically
		if isSourceDist {
			idx := strings.Index(workingDirectory, "/rt/shared")
			if idx > -1 {
				runningArtifactMustBeFromShared = true
			}
		} else {
			idx := strings.Index(workingDirectory, "/relish/shared")
			if idx > -1 {
				runningArtifactMustBeFromShared = true
			}
		}

		if !publish {

			// See if the current directory is a particular artifact version directory,
			// or even is a particular package directory.

			originPos := strings.Index(workingDirectory, "/artifacts/") + 11
			if originPos == 10 {
				originPos = strings.Index(workingDirectory, "/replicas/") + 10
			}
			if originPos >= 10 {
				match := reVersionedPackage.FindStringSubmatchIndex(workingDirectory)
				if match != nil {
					version = workingDirectory[match[2]:match[3]]

					originAndArtifact = workingDirectory[originPos:match[0]]
					packagePath = workingDirectory[match[3]+1:]
				} else {
					match := reVersionAtEnd.FindStringSubmatch(workingDirectory)
					if match != nil {
						version = match[1]
						originAndArtifact = workingDirectory[originPos : len(workingDirectory)-len(version)-2]
					}
				}
			}
		}
	}

	if !publish {
		builtin.InitBuiltinFunctions(relishRoot)
	}

	crypto_util.SetRelishRuntimeLocation(relishRoot) // So that keys can be fetched.

	if publish {
		if len(pathParts) < 2 {
			fmt.Println("Usage (example): relish -publish someorigin.com2013/artifact_name 1.0.23")
			return
		}
		originAndArtifact = pathParts[0]
		version = pathParts[1]

		if strings.HasSuffix(originAndArtifact, "/") { // Strip trailing / if present
			originAndArtifact = originAndArtifact[:len(originAndArtifact)-1]
		}

		err = global_publisher.PublishSourceCode(relishRoot, originAndArtifact, version)
		if err != nil {
			fmt.Println(err)
		}
		return
	}

	sourceCodeShareDir := ""
	if shareListeningPort != 0 {
		// sourceCodeShareDir hould be the "relish/shared"
		// or "relish/rt/shared" of "relish/4production/shared" or "relish/rt/4production/shared" directory.
		sourceCodeShareDir = relishRoot + "/shared"
	}
	onlyCodeSharing := (shareListeningPort != 0 && webListeningPort == 0 && tlsWebListeningPort == 0)

	if onlyCodeSharing {

		if shareListeningPort < 1024 && shareListeningPort != 80 {
			fmt.Println("Error: The source-code sharing port must be 80 or > 1023 (8421 is the standard if using a high port)")
			return
		}

		web.ListenAndServeSourceCode(shareListeningPort, sourceCodeShareDir)

		// The previous call should wait on incoming socket connections on the specified port, and never return.

		// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
		// have permission to listen on a < 1024 port like port 80.
		fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
			"Did you setup to give relish permission to bind to privileged ports?\n"+
			"or is another process already listening on this port?\n", shareListeningPort)
		return
	}

	var loader = global_loader.NewLoader(relishRoot, sharedCodeOnly, dbName+".db", quiet)

	if originAndArtifact == "" {

		if len(pathParts) == 3 { // originAndArtifact version packagePath

			originAndArtifact = pathParts[0]
			version = pathParts[1]
			packagePath = pathParts[2]

		} else if len(pathParts) == 2 { // originAndArtifact packagePath

			originAndArtifact = pathParts[0]
			packagePathOrVersion := pathParts[1]

			// Determine if is a version by regexp or a package path has been supplied

			version = reVersion.FindString(packagePathOrVersion)
			if version == "" {
				packagePath = packagePathOrVersion

			}
		} else if shareListeningPort == 0 || webListeningPort != 0 || tlsWebListeningPort != 0 {
			if len(pathParts) != 1 {
				fmt.Println("Usage: relish [-web 80] originAndArtifact [version] [path/to/package]\n# package path defaults to main")
				return
			}
			originAndArtifact = pathParts[0]
		}
	} else if packagePath == "" {
		if len(pathParts) == 1 {
			packagePath = pathParts[0]
		} else if len(pathParts) > 1 {
			fmt.Println("Usage (when in an artifact version directory): relish [-web 80] [path/to/package]\n# package path defaults to main")
			return
		}
	} else { // both originAndArtifact and packagePath are defined (non "")
		if len(pathParts) != 0 {
			fmt.Println("Usage (when in a package directory): relish [-web 80]")
			return
		}
	}

	if strings.HasSuffix(originAndArtifact, "/") { // Strip trailing / if present
		originAndArtifact = originAndArtifact[:len(originAndArtifact)-1]
	}
	if strings.HasSuffix(packagePath, "/") { // Strip trailing / if present
		packagePath = packagePath[:len(packagePath)-1]
	}

	if packagePath == "" {
		packagePath = "main" // substitute a default.
	}

	fullPackagePath := fmt.Sprintf("%s/v%s/pkg/%s", originAndArtifact, version, packagePath)
	fullUnversionedPackagePath := fmt.Sprintf("%s/pkg/%s", originAndArtifact, packagePath)

	g, err = loader.LoadPackage(originAndArtifact, version, packagePath, runningArtifactMustBeFromShared)

	if err != nil {
		if version == "" {
			fmt.Printf("Error loading package %s from current version of %s:  %v\n", packagePath, originAndArtifact, err)
		} else {
			fmt.Printf("Error loading package %s:  %v\n", fullPackagePath, err)
		}
		return
	}

	g.Interp.SetRunningArtifact(originAndArtifact)

	g.Interp.SetPackageLoader(loader)

	// TODO the following rather twisty logic  (from here to end of main method) could be straightened out.
	// One of its purposes is to ensure that the last http listener is run in this goroutine rather
	// than in a background one. And there can be different numbers of listeners...

	// Count how many separate listeners there will be.

	numListeners := 0
	numListening := 0 // how many of those are already listening?

	if webListeningPort != 0 {
		numListeners += 1
		if shareListeningPort != 0 && shareListeningPort != webListeningPort {
			numListeners += 1
		}
	}
	if tlsWebListeningPort != 0 {
		numListeners += 1
	}
	if explorerListeningPort != 0 {
		numListeners += 1
	}

	// end counting listeners

	// check for disallowed port numbers, and if not, load the packages needed for web app serving

	if webListeningPort != 0 {
		if webListeningPort < 1024 && webListeningPort != 80 {
			fmt.Println("Error: The web listening port must be 80 or > 1023")
			return
		}

		if shareListeningPort != webListeningPort && shareListeningPort != 0 && shareListeningPort < 1024 && shareListeningPort != 80 {
			fmt.Println("Error: The source-code sharing port must be 80 or > 1023 (8421 is the standard if using a high port)")
			return
		}
	}

	if tlsWebListeningPort != 0 {
		if tlsWebListeningPort < 1024 && tlsWebListeningPort != 443 {
			fmt.Println("Error: The tls web listening port must be 443 or > 1023")
			return
		}
	}

	if webListeningPort != 0 || tlsWebListeningPort != 0 {
		err = loader.LoadWebPackages(originAndArtifact, version, runningArtifactMustBeFromShared)
		if err != nil {
			if version == "" {
				fmt.Printf("Error loading web packages from current version of %s:  %v\n", originAndArtifact, err)
			} else {
				fmt.Printf("Error loading web packages from version %s of %s:  %v\n", version, originAndArtifact, err)
			}
			return
		}
	}

	// check for disallowed port numbers, and if not, load the package needed for explorer_api web service serving

	if explorerListeningPort != 0 {
		if explorerListeningPort < 1024 && explorerListeningPort != 80 {
			fmt.Println("Error: The explorer listening port must be 80 or > 1023")
			return
		}

		explorerApiOriginAndArtifact := "shared.relish.pl2012/explorer_api"
		explorerApiPackagePath := "web"
		_, err = loader.LoadPackage(explorerApiOriginAndArtifact, "", explorerApiPackagePath, false)

		if err != nil {
			fmt.Printf("Error loading package %s from current version of %s:  %v\n",
				explorerApiPackagePath,
				explorerApiOriginAndArtifact,
				err)
			return
		}
	}

	if numListeners > 0 { // If we'll be listening for http requests, run main in a background goroutine.
		go g.Interp.RunMain(fullUnversionedPackagePath, quiet)
	}

	if webListeningPort != 0 || tlsWebListeningPort != 0 {

		web.SetWebPackageSrcDirPath(loader.PackageSrcDirPath(originAndArtifact + "/pkg/web"))

		if webListeningPort != 0 {
			if shareListeningPort == webListeningPort {
				numListening += 1
				if numListening == numListeners {
					web.ListenAndServe(webListeningPort, sourceCodeShareDir)
					// The previous call should wait on incoming socket connections on the specified port, and never return.

					// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
					// have permission to listen on a < 1024 port like port 80.
					fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
						"Did you setup to give relish permission to bind to privileged ports?\n"+
						"or is another process already listening on this port?\n", webListeningPort)
					return
				} else {
					go web.ListenAndServe(webListeningPort, sourceCodeShareDir)
				}
			} else {
				if shareListeningPort != 0 {
					numListening += 1
					go web.ListenAndServeSourceCode(shareListeningPort, sourceCodeShareDir)
				}

				numListening += 1
				if numListening == numListeners {
					web.ListenAndServe(webListeningPort, "")
					// The previous call should wait on incoming socket connections on the specified port, and never return.

					// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
					// have permission to listen on a < 1024 port like port 80.
					fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
						"Did you setup to give relish permission to bind to privileged ports?\n"+
						"or is another process already listening on this port?\n", webListeningPort)
					return
				} else {
					go web.ListenAndServe(webListeningPort, "")
				}
			}
		}

		if tlsWebListeningPort != 0 {

			tlsCertPath, tlsKeyPath, err := crypto_util.GetTLSwebServerCertAndKeyFilePaths()
			if err != nil {
				fmt.Printf("Error starting TLS web listener: %s\n", err)
				return
			}
			numListening += 1
			if numListening == numListeners {
				web.ListenAndServeTLS(tlsWebListeningPort, tlsCertPath, tlsKeyPath)
				// The previous call should wait on incoming socket connections on the specified port, and never return.

				// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
				// have permission to listen on a < 1024 port like port 80.
				fmt.Printf("Error: Could not bind to port %d to listen for https connections.\n"+
					"Did you setup to give relish permission to bind to privileged ports?\n"+
					"or is another process already listening on this port?\n", tlsWebListeningPort)
				return
			} else {
				go web.ListenAndServeTLS(tlsWebListeningPort, tlsCertPath, tlsKeyPath)
			}
		}
	}

	if explorerListeningPort != 0 {
		web.ListenAndServeExplorerApi(explorerListeningPort)
		// The previous call should wait on incoming socket connections on the specified port, and never return.

		// If get here, we had a PORT binding problem. Perhaps relish (running as current user) does not
		// have permission to listen on a < 1024 port like port 80.
		fmt.Printf("Error: Could not bind to port %d to listen for http connections.\n"+
			"Did you setup to give relish permission to bind to privileged ports?\n"+
			"or is another process already listening on this port?\n", explorerListeningPort)
		return
	}

	// This will only be reached if numListeners == 0 or there is an error starting listeners.
	// but don't want to re-run main if there was an error starting listeners.
	if numListeners == 0 {
		g.Interp.RunMain(fullUnversionedPackagePath, quiet)
	}
}
Пример #6
0
/*
Run the garbage collector loop.

Checks every params.GcIntervalSeconds to see if it should run the GC.
Runs it if the current allocated-and-not-yet-freed memory is greater than twice
the lowest memory allocated-and-not-freed level since GC was last run.

!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!
TODO : !!!!!!!!! : Need to add a params.GcForcedIntervalSeconds e.g. 3 minutes worth
where it will definitely GC if that amount of time has elapsed.
*/
func (i *Interpreter) GCLoop() {

	defer Un(Trace(GC2_, "GCLoop"))

	var prevA uint64

	var prevGcTime time.Time = time.Now()
	var currentGcTime time.Time
	var forced bool
	var nGCs int64

	for {
		time.Sleep(time.Duration(params.GcIntervalSeconds) * time.Second)
		// time.Sleep(4 * time.Second)

		// See if too much time has passed since last relish GC
		forced = false
		currentGcTime = time.Now()
		if currentGcTime.Sub(prevGcTime) > time.Duration(params.GcForceIntervalSeconds)*time.Second {
			forced = true
		}

		runtime.ReadMemStats(&m)
		if forced || (m.Alloc > prevA*2) || (m.Alloc > prevA+10000000) { // if grew double or by more than 10 MB approx
			if forced {
				Logln(GC_, "GC because more than", params.GcForceIntervalSeconds, "seconds since last GC. Prev Alloc", prevA, ", Alloc", m.Alloc)
			} else {
				Logln(GC_, "GC because Prev Alloc", prevA, ", Alloc", m.Alloc)
			}

			i.GC()

			prevGcTime = currentGcTime
			nGCs++
			runtime.ReadMemStats(&m)
			prevA = m.Alloc

			Logln(GC_, "Sys", m.Sys, "Mallocs", m.Mallocs, "Frees", m.Frees)
			Logln(GC_, "HeapAlloc", m.HeapAlloc, "HeapInuse", m.HeapInuse, "HeapIdle", m.HeapIdle, "HeapReleased", m.HeapReleased, "HeapObjects", m.HeapObjects, "HeapSys", m.HeapSys)
			Logln(GC_, "StackInuse", m.StackInuse, "StackSys", m.StackSys)
			if Logging(GC_) {
				i.rt.DebugAttributesMemory()
			}
			if Logging(GC2_) {
				memProfileFilename := fmt.Sprintf("memory%d.prof", nGCs)
				f, err := gos.Create(memProfileFilename)
				if err == nil {
					pprof.WriteHeapProfile(f)
					f.Close()
				} else {
					Logln(GC2_, "Unable to create memory profile file", memProfileFilename)
				}

			}
		} else if m.Alloc < prevA {

			prevA = m.Alloc

		}
	}
}