Пример #1
0
// compileImports takes a File, loads all the imports, and merges them
// into the File.
func (c *Compiler) compileImports(root *File) error {
	// If we have no imports, short-circuit the whole thing
	if len(root.Imports) == 0 {
		return nil
	}

	// Pull these out into variables so they're easier to reference
	storage := c.importStorage
	cache := c.importCache
	cacheLock := &c.importLock

	// A graph is used to track for cycles
	var graphLock sync.Mutex
	graph := new(dag.AcyclicGraph)
	graph.Add("root")

	// Since we run the import in parallel, multiple errors can happen
	// at the same time. We use multierror and a lock to keep track of errors.
	var resultErr error
	var resultErrLock sync.Mutex

	// Forward declarations for some nested functions we use. The docs
	// for these functions are above each.
	var importSingle func(parent string, f *File) bool
	var downloadSingle func(string, *sync.WaitGroup, *sync.Mutex, []*File, int)

	// importSingle is responsible for kicking off the imports and merging
	// them for a single file. This will return true on success, false on
	// failure. On failure, it is expected that any errors are appended to
	// resultErr.
	importSingle = func(parent string, f *File) bool {
		var wg sync.WaitGroup

		// Build the list of files we'll merge later
		var mergeLock sync.Mutex
		merge := make([]*File, len(f.Imports))

		// Go through the imports and kick off the download
		for idx, i := range f.Imports {
			source, err := getter.Detect(
				i.Source, filepath.Dir(f.Path),
				getter.Detectors)
			if err != nil {
				resultErrLock.Lock()
				defer resultErrLock.Unlock()
				resultErr = multierror.Append(resultErr, fmt.Errorf(
					"Error loading import source: %s", err))
				return false
			}

			// Add this to the graph and check now if there are cycles
			graphLock.Lock()
			graph.Add(source)
			graph.Connect(dag.BasicEdge(parent, source))
			cycles := graph.Cycles()
			graphLock.Unlock()
			if len(cycles) > 0 {
				for _, cycle := range cycles {
					names := make([]string, len(cycle))
					for i, v := range cycle {
						names[i] = dag.VertexName(v)
					}

					resultErrLock.Lock()
					defer resultErrLock.Unlock()
					resultErr = multierror.Append(resultErr, fmt.Errorf(
						"Cycle found: %s", strings.Join(names, ", ")))
					return false
				}
			}

			wg.Add(1)
			go downloadSingle(source, &wg, &mergeLock, merge, idx)
		}

		// Wait for completion
		wg.Wait()

		// Go through the merge list and look for any nil entries, which
		// means that download failed. In that case, return immediately.
		// We assume any errors were put into resultErr.
		for _, importF := range merge {
			if importF == nil {
				return false
			}
		}

		for _, importF := range merge {
			// We need to copy importF here so that we don't poison
			// the cache by modifying the same pointer.
			importFCopy := *importF
			importF = &importFCopy
			source := importF.ID
			importF.ID = ""
			importF.Path = ""

			// Merge it into our file!
			if err := f.Merge(importF); err != nil {
				resultErrLock.Lock()
				defer resultErrLock.Unlock()
				resultErr = multierror.Append(resultErr, fmt.Errorf(
					"Error merging import %s: %s", source, err))
				return false
			}
		}

		return true
	}

	// downloadSingle is used to download a single import and parse the
	// Appfile. This is a separate function because it is generally run
	// in a goroutine so we can parallelize grabbing the imports.
	downloadSingle = func(source string, wg *sync.WaitGroup, l *sync.Mutex, result []*File, idx int) {
		defer wg.Done()

		// Read from the cache if we have it
		cacheLock.Lock()
		cached, ok := cache[source]
		cacheLock.Unlock()
		if ok {
			log.Printf("[DEBUG] cache hit on import: %s", source)
			l.Lock()
			defer l.Unlock()
			result[idx] = cached
			return
		}

		// Call the callback if we have one
		log.Printf("[DEBUG] loading import: %s", source)
		if c.opts.Callback != nil {
			c.opts.Callback(&CompileEventImport{
				Source: source,
			})
		}

		// Download the dependency
		if err := storage.Get(source, source, true); err != nil {
			resultErrLock.Lock()
			defer resultErrLock.Unlock()
			resultErr = multierror.Append(resultErr, fmt.Errorf(
				"Error loading import source: %s", err))
			return
		}
		dir, _, err := storage.Dir(source)
		if err != nil {
			resultErrLock.Lock()
			defer resultErrLock.Unlock()
			resultErr = multierror.Append(resultErr, fmt.Errorf(
				"Error loading import source: %s", err))
			return
		}

		// Parse the Appfile
		importF, err := ParseFile(filepath.Join(dir, "Appfile"))
		if err != nil {
			resultErrLock.Lock()
			defer resultErrLock.Unlock()
			resultErr = multierror.Append(resultErr, fmt.Errorf(
				"Error parsing Appfile in %s: %s", source, err))
			return
		}

		// We use the ID to store the source, but we clear it
		// when we actually merge.
		importF.ID = source

		// Import the imports in this
		if !importSingle(source, importF) {
			return
		}

		// Once we're done, acquire the lock and write it
		l.Lock()
		result[idx] = importF
		l.Unlock()

		// Write this into the cache.
		cacheLock.Lock()
		cache[source] = importF
		cacheLock.Unlock()
	}

	importSingle("root", root)
	return resultErr
}
Пример #2
0
// compileImports需要一个文件,load所有的imports,然后合并到文件中
func compileImports(root *File, importOpts *compileImportOpts, opts *CompileOpts) error {
	// 如果没有imports,直接短路(返回)
	if len(root.Imports) == 0 {
		return nil
	}

	// 把它们放入变量,以便我们可以更早的引用
	//storage := importOpts.Storage
	//cache := importOpts.Cache
	//cacheLock := importOpts.CacheLock

	// A graph is used to track for cycles
	var graphLock sync.Mutex
	graph := new(dag.AcyclicGraph)
	graph.Add("root")

	// 自从我们平行的执行导入,在同一时间会有多个error发生
	// 我们使用multierror lock和跟踪error
	var resultErr error
	var resultErrLock sync.Mutex

	// Forward declarations for some nested functions we use. The docs
	// for these functions are above each.
	var importSingle func(parent string, f *File) bool
	var downloadSingle func(string, *sync.WaitGroup, *sync.Mutex, []*File, int)

	// importSingle is responsible for kicking off the imports and merging
	// them for a single file. This will return true on success, false on
	// failure. On failure, it is expected that any errors are appended to
	// resultErr.
	importSingle = func(parent string, f *File) bool {
		var wg sync.WaitGroup

		// 构建文件列表,后面将合并
		var mergeLock sync.Mutex
		merge := make([]*File, len(f.Imports))

		// 通过导入并开始处理下载
		for idx, i := range f.Imports {
			source, err := getter.Detect(i.Source, filepath.Dir(f.Path), getter.Detectors)
			if err != nil {
				resultErrLock.Lock()
				defer resultErrLock.Unlock()
				resultErr = multierror.Append(resultErr, fmt.Errorf(
					"获取import源错误: %s", err))
				return false
			}

			// Add this to the graph and check now if there are cycles
			graphLock.Lock()
			graph.Add(source)
			graph.Connect(dag.BasicEdge(parent, source))
			cycles := graph.Cycles()
			graphLock.Unlock()
			if len(cycles) > 0 {
				for _, cycle := range cycles {
					names := make([]string, len(cycle))
					for i, v := range cycle {
						names[i] = dag.VertexName(v)
					}

					resultErrLock.Lock()
					defer resultErrLock.Unlock()
					resultErr = multierror.Append(resultErr, fmt.Errorf(
						"Cycle found: %s", strings.Join(names, ", ")))
					return false
				}
			}
			wg.Add(1)
			go downloadSingle(source, &wg, &mergeLock, merge, idx)
		}

		// Wait for completion
		wg.Wait()

		// Go through the merge list and look for any nil entries, which
		// means that download failed. In that case, return immediately.
		// We assume any errors were put into resultErr.
		for _, importF := range merge {
			if importF == nil {
				return false
			}
		}

		for _, importF := range merge {
			// We need to copy importF here so that we don't poison
			// the cache by modifying the same pointer.
			importFCopy := *importF
			importF = &importFCopy
			source := importF.ID
			importF.ID = ""
			importF.Path = ""

			// Merge it into our file!
			if err := f.Merge(importF); err != nil {
				resultErrLock.Lock()
				defer resultErrLock.Unlock()
				resultErr = multierror.Append(resultErr, fmt.Errorf(
					"合并import错误 %s : %s", source, err))
				return false
			}
		}
		return true
	}

	// downloadSingle is used to download a single import and parse the
	// Appfile. This is a separate function because it is generally run
	// in a goroutine so we can parallelize grabbing the imports.
	downloadSingle = func(source string, wg *sync.WaitGroup, l *sync.Mutex, result []*File, idx int) {}

	importSingle("root", root)
	return resultErr
}