Beispiel #1
0
func handle_json_req(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	var body_reader io.Reader = r.Body

	b := bytes.Buffer{}
	n, e := b.ReadFrom(body_reader)
	_ = n
	if e != nil {
		send_error_bad_request(w, fmt.Sprintf("%v", e))
		return
	}

	if g_req_debug {
		log.Printf(b.String())
	}

	sj, e := sloppyjson.Loads(b.String())
	if e != nil {
		send_error_bad_request(w, fmt.Sprintf("%v", e))
		return
	}

	conn_info := LanternConnInfo{}
	conn_info.UUID, e = gouuid.NewV4()
	if e != nil {
		send_error_bad_request(w, fmt.Sprintf("%v", e))
		return
	}

	if _, ok := sj.O["Type"]; !ok {
		send_error_bad_request(w, fmt.Sprintf("%v", e))
		return
	}

	if gVerboseFlag {
		log.Printf("%s: received %s\n", conn_info.UUID.String(), sj.O["Type"].S)
	}

	switch sj.O["Type"].S {
	case "ping":
		ping_req_handler(w, sj, &conn_info)
		return
	case "stat":
		stat_req_handler(w, sj, &conn_info)
		return
	}

}
Beispiel #2
0
func _load_json_config(ctx *LanternContext, config_fn string) error {
	var e error

	raw_str, err := ioutil.ReadFile(config_fn)
	if err != nil {
		return err
	}

	//gConfig,e = sloppyjson.Loads(string(raw_str))
	ctx.Config, e = sloppyjson.Loads(string(raw_str))
	if e != nil {
		return e
	}

	log.Printf("config loaded\n")

	//gConfig.Printr(0,2)

	return nil
}
// Open a stream and read the FastJ file.
// Populate g_tile_lib.  This will group
// tiles by path.step in g_tile_lib.  In
// each grouping there will be a tile per md5sum
// with the appropriate TileInfo field.
//
func import_fastj(name, fn string) error {
	var prev_md5sum string
	var prev_tile_path string
	var prev_tileid string
	_ = prev_tileid
	var prev_seedlen int

	var prev_path_i int64
	var prev_step_i int64

	curseq := make([]string, 0, 10)

	h, e := autoio.OpenReadScannerSimple(fn)
	if e != nil {
		return e
	}
	defer h.Close()

	for h.ReadScan() {
		l := h.ReadText()
		if len(l) == 0 {
			continue
		}

		if l[0] == '>' {
			sj, e := sloppyjson.Loads(l[1:])
			if e != nil {
				return e
			}

			md5sum := sj.O["md5sum"].S
			tileid := sj.O["tileID"].S
			seedlen := int(sj.O["seedTileLength"].P)

			tile_parts := strings.SplitN(tileid, ".", 4)
			tile_path := fmt.Sprintf("%s.%s", tile_parts[0], tile_parts[2])

			path_i, e := strconv.ParseInt(tile_parts[0], 16, 64)
			if e != nil {
				return e
			}
			step_i, e := strconv.ParseInt(tile_parts[2], 16, 64)
			if e != nil {
				return e
			}

			if _, ok := g_path_md5sum_freq[tile_path]; !ok {
				g_path_md5sum_freq[tile_path] = make(map[string]int)
			}
			g_path_md5sum_freq[tile_path][md5sum]++

			pfx := "0:"
			if tile_parts[3] == "001" {
				pfx = "1:"
			}
			g_path_md5sum[tile_path] = append(g_path_md5sum[tile_path], pfx+md5sum)

			if len(curseq) > 0 {

				tile_seq := strings.Join(curseq, "")
				pfx_tag := tile_seq[0:24]
				sfx_tag := tile_seq[len(tile_seq)-24:]

				md5_tile_seq := md5sum_str(tile_seq)
				if md5_tile_seq != prev_md5sum {
					log.Fatal(fmt.Sprintf("previous md5sum %s (%s) != current md5sum %s (%s)\n", prev_md5sum, tileid, md5_tile_seq, prev_tileid))
				}

				if _, ok := g_md5sum_seq[md5_tile_seq]; !ok {
					g_md5sum_seq[md5_tile_seq] = tile_seq
				}

				pfx_tag_id := create_tag_id(prev_tile_path, pfx_tag)
				if _, ok := g_id_tag[pfx_tag_id]; !ok {
					g_id_tag[pfx_tag_id] = pfx_tag
				}

				//sfx_tag_id := create_tag_id(prev_tile_path, sfx_tag)
				prev_sfx_tile_path := fmt.Sprintf("%03x.%04x", prev_path_i, prev_step_i+int64(prev_seedlen))
				sfx_tag_id := create_tag_id(prev_sfx_tile_path, sfx_tag)

				if _, ok := g_id_tag[sfx_tag_id]; !ok {
					g_id_tag[sfx_tag_id] = sfx_tag
				}

				if _, ok := g_tile_lib[prev_tile_path]; !ok {
					g_tile_lib[prev_tile_path] = make(map[string]TileInfo)
				}

				if _, ok := g_tile_lib[prev_tile_path][prev_md5sum]; !ok {
					g_tile_lib[prev_tile_path][prev_md5sum] = TileInfo{prev_md5sum, prev_tile_path, prev_seedlen, 1, -1}
				} else {
					z := g_tile_lib[prev_tile_path][prev_md5sum]
					z.Freq++
					g_tile_lib[prev_tile_path][prev_md5sum] = z
				}

			} else {
			}

			curseq = curseq[0:0]

			prev_md5sum = md5sum
			prev_tile_path = tile_path
			prev_tileid = tileid
			prev_seedlen = seedlen
			prev_path_i = path_i
			prev_step_i = step_i

			continue

		}

		curseq = append(curseq, l)

	}

	if len(curseq) > 0 {
		tile_seq := strings.Join(curseq, "")
		pfx_tag := tile_seq[0:24]
		sfx_tag := tile_seq[len(tile_seq)-24:]

		md5_tile_seq := md5sum_str(tile_seq)
		if md5_tile_seq != prev_md5sum {
			log.Fatal(fmt.Sprintf("previous md5sum %s != current md5sum %s (%s)\n", prev_md5sum, md5_tile_seq, prev_tileid))
		}

		if _, ok := g_md5sum_seq[md5_tile_seq]; !ok {
			g_md5sum_seq[md5_tile_seq] = tile_seq
		}

		pfx_tag_id := create_tag_id(prev_tile_path, pfx_tag)
		if _, ok := g_id_tag[pfx_tag_id]; !ok {
			g_id_tag[pfx_tag_id] = pfx_tag
		}

		//sfx_tag_id := create_tag_id(prev_tile_path, sfx_tag)
		prev_sfx_tile_path := fmt.Sprintf("%03x.%04x", prev_path_i, prev_step_i+int64(prev_seedlen))
		sfx_tag_id := create_tag_id(prev_sfx_tile_path, sfx_tag)

		if _, ok := g_id_tag[sfx_tag_id]; !ok {
			g_id_tag[sfx_tag_id] = sfx_tag
		}

		if _, ok := g_tile_lib[prev_tile_path]; !ok {
			g_tile_lib[prev_tile_path] = make(map[string]TileInfo)
		}

		if _, ok := g_tile_lib[prev_tile_path][prev_md5sum]; !ok {
			g_tile_lib[prev_tile_path][prev_md5sum] = TileInfo{prev_md5sum, prev_tile_path, prev_seedlen, 1, -1}
		} else {
			z := g_tile_lib[prev_tile_path][prev_md5sum]
			z.Freq++
			g_tile_lib[prev_tile_path][prev_md5sum] = z
		}

	}

	_ = name
	return nil

}
// Open a stream and read the FastJ file.
// Populate g_tile_lib.  This will group
// tiles by path.step in g_tile_lib.  In
// each grouping there will be a tile per md5sum
// with the appropriate TileInfo field.
//
func import_fastj(name, fn string) error {
	var prev_md5sum string
	var prev_tile_path string
	_ = prev_tile_path
	var prev_tileid string
	_ = prev_tileid
	var prev_seedlen int
	_ = prev_seedlen

	var prev_path_i int64
	_ = prev_path_i
	var prev_step_i int64
	_ = prev_step_i

	var prev_tile_allele int64
	_ = prev_tile_allele
	var prev_allele_name_id string

	curseq := make([]string, 0, 10)

	h, e := autoio.OpenReadScannerSimple(fn)
	if e != nil {
		return e
	}

	for h.ReadScan() {
		l := h.ReadText()
		if len(l) == 0 {
			continue
		}

		if l[0] == '>' {
			sj, e := sloppyjson.Loads(l[1:])
			if e != nil {
				return e
			}

			md5sum := sj.O["md5sum"].S
			tileid := sj.O["tileID"].S
			seedlen := int(sj.O["seedTileLength"].P)

			tile_parts := strings.SplitN(tileid, ".", 4)
			tile_path := fmt.Sprintf("%s.%s", tile_parts[0], tile_parts[2])

			path_i, e := strconv.ParseInt(tile_parts[0], 16, 64)
			if e != nil {
				return e
			}
			step_i, e := strconv.ParseInt(tile_parts[2], 16, 64)
			if e != nil {
				return e
			}

			tile_allele, e := strconv.ParseInt(tile_parts[3], 16, 64)
			if e != nil {
				return e
			}

			allele_name_id := fmt.Sprintf("%s:%d", name, tile_allele)

			// Initialize everything if we haven't seen it before
			//
			if _, ok := g_allele[allele_name_id]; !ok {

				callset_id := g_callset[name].Id
				ploidy := 1
				variant_set_id := g_START_VARIANTSET_ID

				g_allele[allele_name_id] = Allele{g_ALLELE_ID, variant_set_id, allele_name_id, 0}
				g_allele_path_item[allele_name_id] = make([]AllelePathItem, 0, 1024)
				g_allele_call[allele_name_id] = AlleleCall{g_ALLELE_ID, callset_id, ploidy}
				g_ALLELE_ID++
			}

			if len(curseq) > 0 {

				tile_seq := strings.Join(curseq, "")
				pfx_tag := tile_seq[0:24]
				sfx_tag := tile_seq[len(tile_seq)-24:]
				body_seq := tile_seq[24 : len(tile_seq)-24]

				md5_tile_seq := md5sum_str(tile_seq)
				if md5_tile_seq != prev_md5sum {
					log.Fatal(fmt.Sprintf("previous md5sum %s (%s) != current md5sum %s (%s)\n", prev_md5sum, tileid, md5_tile_seq, prev_tileid))
				}

				var ok bool
				var seqid int64

				allele_id := g_allele[prev_allele_name_id].Id
				allele_path := g_allele_path_item[prev_allele_name_id]
				cur_idx := len(allele_path)

				pfx_md5 := md5sum_str(pfx_tag)
				if seqid, ok = g_md5_seqid_map[pfx_md5]; !ok {
					log.Fatal(fmt.Sprintf("ERROR: could not find tag '%s' (%s) in Sequence map", pfx_tag, pfx_md5))
				}

				if cur_idx == 0 {
					allele_path = append(allele_path, AllelePathItem{allele_id, cur_idx, int(seqid), 0, 24, "'TRUE'"})
					cur_idx++
				}

				body_md5 := md5sum_str(body_seq)
				if seqid, ok = g_md5_seqid_map[body_md5]; !ok {
					log.Fatal(fmt.Sprintf("ERROR: could not find body (%s) in Sequence map", body_md5))
				}

				allele_path = append(allele_path, AllelePathItem{allele_id, cur_idx, int(seqid), 0, len(body_seq), "'TRUE'"})
				cur_idx++

				sfx_md5 := md5sum_str(sfx_tag)
				if seqid, ok = g_md5_seqid_map[sfx_md5]; !ok {
					log.Fatal(fmt.Sprintf("ERROR: could not find tag '%s' (%s) in Sequence map", sfx_tag, sfx_md5))
				}

				allele_path = append(allele_path, AllelePathItem{allele_id, cur_idx, int(seqid), 0, 24, "'TRUE'"})
				cur_idx++

				g_allele_path_item[prev_allele_name_id] = allele_path

			} else {
			}

			curseq = curseq[0:0]

			prev_md5sum = md5sum
			prev_tile_path = tile_path
			prev_tileid = tileid
			prev_seedlen = seedlen
			prev_path_i = path_i
			prev_step_i = step_i
			prev_tile_allele = tile_allele
			prev_allele_name_id = allele_name_id

			continue

		}

		curseq = append(curseq, l)

	}

	if len(curseq) > 0 {
		tile_seq := strings.Join(curseq, "")
		pfx_tag := tile_seq[0:24]
		sfx_tag := tile_seq[len(tile_seq)-24:]
		body_seq := tile_seq[24 : len(tile_seq)-24]

		md5_tile_seq := md5sum_str(tile_seq)
		if md5_tile_seq != prev_md5sum {
			log.Fatal(fmt.Sprintf("previous md5sum %s != current md5sum %s (%s)\n", prev_md5sum, md5_tile_seq, prev_tileid))
		}

		var ok bool
		var seqid int64

		pfx_md5 := md5sum_str(pfx_tag)
		if seqid, ok = g_md5_seqid_map[pfx_md5]; !ok {
			log.Fatal(fmt.Sprintf("ERROR: could not find tag '%s' (%s) in Sequence map", pfx_tag, pfx_md5))
		}

		// Only add the prefix tag if it's the first one in the AllelePathItem
		//
		allele_id := g_allele[prev_allele_name_id].Id
		allele_path := g_allele_path_item[prev_allele_name_id]
		cur_idx := len(allele_path)

		if cur_idx == 0 {
			allele_path = append(allele_path, AllelePathItem{allele_id, cur_idx, int(seqid), 0, 24, "'TRUE'"})
			cur_idx++
		}

		body_md5 := md5sum_str(body_seq)
		if seqid, ok = g_md5_seqid_map[body_md5]; !ok {
			log.Fatal(fmt.Sprintf("ERROR: could not find body (%s) in Sequence map", body_md5))
		}

		allele_path = append(allele_path, AllelePathItem{allele_id, cur_idx, int(seqid), 0, len(body_seq), "'TRUE'"})
		cur_idx++

		sfx_md5 := md5sum_str(sfx_tag)
		if seqid, ok = g_md5_seqid_map[sfx_md5]; !ok {
			log.Fatal(fmt.Sprintf("ERROR: could not find tag '%s' (%s) in Sequence map", sfx_tag, sfx_md5))
		}

		allele_path = append(allele_path, AllelePathItem{allele_id, cur_idx, int(seqid), 0, 24, "'TRUE'"})
		cur_idx++

		g_allele_path_item[prev_allele_name_id] = allele_path

	}

	_ = name
	return nil

}
func _main(c *cli.Context) {
	g_verboseFlag = c.Bool("Verbose")

	beg_str := c.String("start")
	end_str := c.String("end")

	if len(beg_str) > 0 {
		parse_filter(beg_str, &g_beg_path, &g_beg_ver, &g_beg_step, &g_beg_variant)
	}

	if len(end_str) > 0 {
		parse_filter(end_str, &g_end_path, &g_end_ver, &g_end_step, &g_end_variant)
	}

	if len(c.String("input-fastj")) == 0 {
		fmt.Fprintf(os.Stderr, "Provide input FastJ file\n")
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	scanner, err := autoio.OpenReadScannerSimple(c.String("input-fastj"))
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		os.Exit(1)
	}
	defer scanner.Close()

	h_line := ""
	fold_width := 50

	first_pass := true

	line_no := 0
	seq := make([]byte, 300)

	var prev_tileid string
	var prev_seed_tile_len int

	for scanner.ReadScan() {
		line_no++

		l := scanner.ReadText()
		if len(l) == 0 {
			continue
		}
		if l[0] == '>' {

			sj, e := sloppyjson.Loads(l[1:])
			if e != nil {
				log.Fatal(e)
			}

			tileid := sj.O["tileID"].S
			seed_tile_len := int(sj.O["seedTileLength"].P)

			if !first_pass {
				if pass_filter(prev_tileid, prev_seed_tile_len) {
					fmt.Printf("%s\n", h_line)
					p := 0
					for ; p < (len(seq) - fold_width); p += fold_width {
						fmt.Printf("%s\n", seq[p:p+fold_width])
					}
					if p < len(seq) {
						fmt.Printf("%s\n", seq[p:])
					}
					fmt.Printf("\n")
				}
			}

			first_pass = false

			h_line = l
			seq = seq[0:0]
			prev_tileid = tileid
			prev_seed_tile_len = seed_tile_len

			continue
		}

		seq = append(seq, []byte(l)...)
	}

	if !first_pass {
		if pass_filter(prev_tileid, prev_seed_tile_len) {
			fmt.Printf("%s\n", h_line)
			p := 0
			for ; p < (len(seq) - fold_width); p += fold_width {
				fmt.Printf("%s\n", seq[p:p+fold_width])
			}
			if p < len(seq) {
				fmt.Printf("%s\n", seq[p:])
			}
			fmt.Printf("\n")
		}
	}

}
Beispiel #6
0
// Take in a FastJ stream and a reference stream to produce a PASTA stream.
// Assumes each variant 'class' is ordered.
//
func (g *FastJInfo) Pasta(fastj_stream *bufio.Reader, ref_stream *bufio.Reader, assembly_stream *bufio.Reader, out *bufio.Writer) error {
	var err error

	g.LFMod = 50

	for ii := 0; ii < 256; ii++ {
		memz.Score['n'][ii] = 0
		memz.Score[ii]['n'] = 0
	}

	ref_pos := g.RefPos
	ref_seq := make([]byte, 0, 1024)
	alt_seq := make([][]byte, 2)
	alt_seq[0] = make([]byte, 0, 1024)
	alt_seq[1] = make([]byte, 0, 1024)
	tile_len := make([]int, 2)

	is_eof := false

	cur_path := make([]int, 2)
	_ = cur_path
	cur_step := make([]int, 2)
	_ = cur_step
	cur_var := 0

	// For spanning tiles we need to skip the
	// tag at the beginning.  This holds the
	// number of bases we need to skip.
	//
	skip_prefix := make([]int, 2)
	skip_prefix[0] = 0
	skip_prefix[1] = 0

	knot_len := make([]int, 2)
	knot_len[0] = 0
	knot_len[1] = 0

	for {

		line, e := fastj_stream.ReadBytes('\n')
		if e != nil {
			err = e
			if e == io.EOF {
				is_eof = true
			}
			break
		}

		if len(line) == 0 {
			continue
		}
		if line[0] == '\n' {
			continue
		}

		// Beginning of a header line means we can emit the previous tile information.
		//
		if line[0] == '>' {

			if tile_len[0] == tile_len[1] {
				if len(ref_seq) > 24 {

					n := len(ref_seq) - 24
					n0 := len(alt_seq[0]) - 24
					n1 := len(alt_seq[1]) - 24

					if n >= 24 {
						g.EmitAlignedInterleave(ref_seq[:n], alt_seq[0][:n0], alt_seq[1][:n1], out)
					} else {
						return fmt.Errorf("sanity error, no tag")
					}

				}

				tile_len[0] = 0
				tile_len[1] = 0

				skip_prefix[0] = 0
				skip_prefix[1] = 0

				knot_len[0] = 0
				knot_len[1] = 0

				for aa := 0; aa < 2; aa++ {
					n := len(alt_seq[aa])
					if n > 24 {
						alt_seq[aa] = alt_seq[aa][0:0]
					} else {
						alt_seq[aa] = alt_seq[aa][0:0]
					}
				}

				n := len(ref_seq)
				if n > 24 {
					ref_seq = ref_seq[n-24:]
				} else {
					ref_seq = ref_seq[0:0]
				}

			}

			sj, e := sloppyjson.Loads(string(line[1:]))
			if e != nil {
				return fmt.Errorf(fmt.Sprintf("error parsing JSON header: %v", e))
			}

			p, _, s, v, e := parse_tile(sj.O["tileID"].S)
			if e != nil {
				return fmt.Errorf(fmt.Sprintf("error parsing tileID: %v", e))
			}
			_ = p
			_ = s

			stl := int(sj.O["seedTileLength"].P)
			tile_len[v] += stl

			skip_prefix[v] = 0
			if knot_len[v] > 0 {
				skip_prefix[v] = 24
			}
			knot_len[v]++

			cur_var = v

			// Read up to current assembly position in reference and
			// assembly streams.
			//
			if cur_var == 0 {

				for ii := 0; ii < stl; ii++ {

					// Advance the next refere position end, reading as many
					// spanning tiles as we need to (reading 'stl' (seedTileLength)
					// as many entries from the assembly stream).
					//
					e = g.ReadAssembly(assembly_stream)
					if e != nil {
						return fmt.Errorf(fmt.Sprintf("ERROR reading assembly at ref_pos %d: %v", ref_pos, e))
					}

					for {

						if ref_pos >= g.AssemblyEndPos {
							break
						}

						ref_ch, e := ref_stream.ReadByte()
						if e != nil {
							return fmt.Errorf(fmt.Sprintf("error reading reference stream (ref_pos %d, AssemblyEndPos %d): %v", ref_pos, g.AssemblyEndPos, e))
						}
						if ref_ch == '\n' || ref_ch == ' ' || ref_ch == '\t' || ref_ch == '\r' {
							continue
						}

						if ref_ch == '>' {
							msg, e := pasta.ControlMessageProcess(ref_stream)
							if e != nil {
								return fmt.Errorf(fmt.Sprintf("error processing control message: %v", e))
							}
							if msg.Type == pasta.POS {
								ref_pos = msg.RefPos
							}
							continue
						}

						ref_seq = append(ref_seq, ref_ch)
						ref_pos++
					}

					if ref_pos != g.AssemblyEndPos {
						return fmt.Errorf("reference position mismatch")
					}

				}

			}

			continue
		}

		line = bytes.Trim(line, " \t\n")

		if tile_len[cur_var] == 0 {
			alt_seq[cur_var] = append(alt_seq[cur_var], line...)
		} else {

			// Skip the appropriate bases if this is
			// part of a knot.
			//
			min_pfx := skip_prefix[cur_var]
			if min_pfx > len(line) {
				min_pfx = len(line)
			}

			alt_seq[cur_var] = append(alt_seq[cur_var], line[min_pfx:]...)

			// Update bases to skip
			//
			skip_prefix[cur_var] -= min_pfx
		}

	}

	if !is_eof {
		return fmt.Errorf(fmt.Sprintf("non EOF state after stream processed: %v", err))
	}

	// Take care of final tiles
	//
	if tile_len[0] == tile_len[1] {

		if len(ref_seq) >= 24 {
			g.EmitAlignedInterleave(ref_seq, alt_seq[0], alt_seq[1], out)
		} else {
			return fmt.Errorf("sanity, no tag")
		}

	} else {
		return fmt.Errorf("tile position mismatch")
	}

	out.WriteByte('\n')
	out.Flush()

	return nil
}