Exemplo n.º 1
0
			if err != nil {
				panic(err)
			}
		}

		getElapsedTime(&startTime)
	},
}

var cmdExtractRecipiesFromUrl = &cobra.Command{
	Use:   "process-url",
	Short: "Extract recipes from the given URL",
	Run: func(cmd *cobra.Command, args []string) {
		startTime = time.Now()

		recipes := r.DownloadRecipesFromUrls(args).Recipes

		fmt.Printf("Found %d recipes.\n", len(recipes))
		for i, recipe := range recipes {
			fmt.Printf("Recipe #%d: %s\n", i, recipe.String())
		}

		if globalConfig.MongoUrl != "" {
			err := r.SaveRecipes(globalConfig.MongoUrl, recipes)
			if err != nil {
				panic(err)
			}
		}

		getElapsedTime(&startTime)
	},
Exemplo n.º 2
0
				for _, url := range unprocessedFilteredURLs {
					if !existingURLsTable[url] {
						processableURLs = append(processableURLs, url)
					}
				}

				fmt.Printf("Found existing recipes for %d of %d URLs. Will process %d URLs.\n", len(existingURLs), len(unprocessedFilteredURLs), len(processableURLs))
				for _, url := range existingURLs {
					fmt.Printf("Existing:     %s\n", url)
				}
				for _, url := range processableURLs {
					fmt.Printf("Will process: %s\n", url)
				}
			}

			result := r.DownloadRecipesFromUrls(processableURLs)

			pageStats := RecipeStats{}
			pageStats.RecipeCount = len(result.Recipes)
			pageStats.ArticlesWithoutRecipesCount = len(result.URLsWithoutRecipes)
			pageStats.articlesProcessedCount = len(result.URLs)
			pageStats.articlesIgnoredAsDuplicatesCount = len(filteredURLs) - len(unprocessedFilteredURLs)
			pageStats.articlesIgnoredAsExistingCount = len(unprocessedFilteredURLs) - len(processableURLs)
			pageStats.articlesIgnoredInWrongSectionCount = len(urls) - len(filteredURLs)
			pageStats.URLsWithoutRecipes = result.URLsWithoutRecipes
			pageStats.TotalURLsWithRecipesCount = (pageStats.articlesProcessedCount - pageStats.ArticlesWithoutRecipesCount) + pageStats.articlesIgnoredAsExistingCount

			overallStats.merge(pageStats)

			for _, url := range unprocessedFilteredURLs {
				processedURLsTable[url] = true