Example #1
0
func main() {
	// PETSc initialization
	if err := petsc.Initialize(); err != nil {
		petsc.Fatal(err)
	}
	defer func() {
		if err := petsc.Finalize(); err != nil {
			petsc.Fatal(err)
		}
	}()
	rank, size := petsc.RankSize()

	// Create particles
	var np1 int64 = 1
	if rank == 0 {
		np1 = 2
	}
	pp, err := structvec.NewStructVec(pstruct{}, np1, petsc.DETERMINE)
	if err != nil {
		petsc.Fatal(err)
	}
	defer pp.Destroy()

	lpp, _ := pp.GetArray().([]pstruct)
	for i := range lpp {
		for j := 0; j < 3; j++ {
			lpp[i].pos[j] = (float32(i) + 1) * (float32(j + 1 + rank*10))
		}
	}
	pp.RestoreArray()

	lpp, _ = pp.GetArray().([]pstruct)
	dump(lpp, rank)
	pp.RestoreArray()

	// Set up scatters
	var localndx, mpirank []int64
	if rank == 0 {
		localndx = make([]int64, 1+size)
		mpirank = make([]int64, 1+size)
		for i := 0; i < size; i++ {
			localndx[i] = 0
			mpirank[i] = int64(i)
		}
		localndx[size] = 1
		mpirank[size] = int64((rank + 1) % size)
	} else {
		localndx = make([]int64, 1)
		mpirank = make([]int64, 1)
		localndx[0] = 0
		mpirank[0] = int64((rank + 1) % size)
	}

	petsc.Printf("\n\n\n")
	pp.Scatter(localndx, mpirank)
	lpp, _ = pp.GetArray().([]pstruct)
	dump(lpp, rank)
	pp.RestoreArray()

}
Example #2
0
func main() {
	if err := petsc.Initialize(); err != nil {
		petsc.Fatal(err)
	}
	defer func() {
		if err := petsc.Finalize(); err != nil {
			petsc.Fatal(err)
		}
	}()
	rank, size := petsc.RankSize()

	petsc.Printf("Initialization successful\n")
	petsc.SyncPrintf("Hello from rank %d of %d\n", rank, size)
	petsc.SyncFlush()
}
Example #3
0
func NewVec(nlocal, nglobal int64) *structvec.StructVec {
	v, err := structvec.NewStructVec(One{}, nlocal, nglobal)
	if err != nil {
		petsc.Fatal(err)
	}
	return v
}
Example #4
0
// LocalSizeTransposed returns the local size required for the transform,
// as well as the start and end positions for the transposed and untransposed
// dimensions.
//
// Unlike the FFTW3 interface, dims here are the dimensions for the real transform.
//
// returns the local size (number of real elts), n0 and n1 start and local sizes.
func LocalSizeTransposed(dims []int64) (int64, [2]int64, [2]int64) {
	ndim := len(dims)
	if ndim < 2 {
		petsc.Fatal(errors.New("Need at least two dimensions"))
	}

	rnk := C.int(ndim)
	// We're going to change the dimensions, so make a copy.
	dims1 := make([]int64, ndim)
	copy(dims1, dims)
	dims1[ndim-1] = dims[ndim-1]/2 + 1

	// ptrdiff_t fftw_mpi_local_size_transposed(int rnk, const ptrdiff_t *n, MPI_Comm comm,
	//                                               ptrdiff_t *local_n0, ptrdiff_t *local_0_start,
	//                                               ptrdiff_t *local_n1, ptrdiff_t *local_1_start);

	var n0, n0start, n1, n1start C.ptrdiff_t
	lsize := C.fftw_mpi_local_size_transposed(rnk, (*C.ptrdiff_t)(unsafe.Pointer(&dims1[0])), petsc.WORLD,
		&n0, &n0start, &n1, &n1start)

	ln0 := [2]int64{int64(n0), int64(n0start)}
	ln1 := [2]int64{int64(n1), int64(n1start)}

	return int64(lsize) * 2, ln0, ln1
}
Example #5
0
// Initialize initializes FFTW3 for MPI usage
func Initialize() {
	if C.size_ptrdiff() != 8 {
		petsc.Fatal(errors.New("size(ptrdiff_t) != 8"))
	}

	// Initialize FFTW3
	C.fftw_mpi_init()
}
Example #6
0
func (s *StructVec) GetArray() interface{} {
	if err := s.v.GetArray(); err != nil {
		petsc.Fatal(err)
	}

	// Make sure the lengths match
	if int64(len(s.v.Arr)) != (s.Nlocal * s.bs) {
		petsc.Fatal(errors.New("Unexpected size of array"))
	}

	sliceHeaderIn := (*reflect.SliceHeader)(unsafe.Pointer(&s.v.Arr))
	ptr := reflect.New(reflect.SliceOf(s.t))
	sliceHeaderOut := (*reflect.SliceHeader)(unsafe.Pointer(ptr.Pointer()))
	sliceHeaderOut.Cap = int(s.Nlocal)
	sliceHeaderOut.Len = int(s.Nlocal)
	sliceHeaderOut.Data = sliceHeaderIn.Data

	return ptr.Elem().Interface()
}
Example #7
0
func main() {
	// PETSc initialization
	if err := petsc.Initialize(); err != nil {
		petsc.Fatal(err)
	}
	defer func() {
		if err := petsc.Finalize(); err != nil {
			petsc.Fatal(err)
		}
	}()
	rank, size := petsc.RankSize()

	pp := PW3D.NewVec(petsc.DECIDE, 10000)
	defer pp.Destroy()

	lpp := PW3D.GetArray(pp)
	lpp.FillRandom(1, 1)
	pp.RestoreArray()
	petsc.Printf("Generating random particles....\n")

	slab := particles.Slab{L: 1, N: size, Idim: 0}
	PW3D.DomainDecompose(slab, pp)
	petsc.Printf("Slab decomposition complete\n")

	lpp = PW3D.GetArray(pp)
	_, mpirank := slab.Domain(lpp)
	rank64 := int64(rank)
	petsc.SyncPrintf("# Rank %d has %d particles....\n", rank, lpp.Length())
	for ipart, irank := range mpirank {
		if irank != rank64 {
			petsc.SyncPrintf("ERROR: %d expected, %d placed, %+v\n", rank, irank, lpp[ipart])
		}
	}
	petsc.SyncFlush()
	pp.RestoreArray()

}
Example #8
0
// New returns a new fftw3.Grid of sides dim[] (specified in configuration space)
func New(dims []int64) (greal *Grid, gcmplx *Grid) {
	// Check that dims is big enough
	ndim := len(dims)
	if ndim < 2 {
		petsc.Fatal(errors.New("The grid must at least be 2D"))
	}

	g := new(Grid)
	g.dims = make([]int64, ndim)
	copy(g.dims, dims)

	// Figure out local dimensions
	lsize, n0, n1 := LocalSizeTransposed(dims)

	// Allocate the real and complex grids
	greal = simpleNew(dims, n0, false)
	gcmplx = simpleNew(dims, n1, true)

	// Ok, now allocate arrays and store the same in both places

	_ = lsize
	return
}
Example #9
0
func main() {
	if err := petsc.Initialize(); err != nil {
		petsc.Fatal(err)
	}
	defer func() {
		if err := petsc.Finalize(); err != nil {
			petsc.Fatal(err)
		}
	}()
	rank, size := petsc.RankSize()

	// Create a vector using the local size
	v, err := petsc.NewVec(5, petsc.DETERMINE)
	if err != nil {
		petsc.Fatal(err)
	}
	n1, err := v.LocalSize()
	if err != nil {
		petsc.Fatal(err)
	}
	lo, hi, err := v.OwnRange()
	if err != nil {
		petsc.Fatal(err)
	}
	petsc.SyncPrintf("%d rank has local size %d [%d, %d]\n", rank, n1, lo, hi)
	petsc.SyncFlush()
	err = v.Destroy()
	if err != nil {
		petsc.Fatal(err)
	}

	// Create a vector using the global size
	v, err = petsc.NewVec(petsc.DECIDE, 100)
	if err != nil {
		petsc.Fatal(err)
	}
	n1, err = v.LocalSize()
	if err != nil {
		petsc.Fatal(err)
	}
	lo, hi, err = v.OwnRange()
	if err != nil {
		petsc.Fatal(err)
	}
	petsc.SyncPrintf("%d rank has local size %d [%d, %d]\n", rank, n1, lo, hi)
	petsc.SyncFlush()

	// Set and then access the array
	if err := v.Set(3.1415926); err != nil {
		petsc.Fatal(err)
	}

	// Try running ownershipranges
	if rank == 0 {
		rr, err := v.Ranges()
		if err != nil {
			petsc.Fatal(err)
		}
		fmt.Println(rr)
		if size > 2 {
			ix := []int64{rr[1], rr[2], rr[3]}
			y := []float64{4.14, 5.14, 6.14}
			v.SetValues(ix, y, true)
		}
	}
	v.AssemblyBegin()
	v.AssemblyEnd()

	if err := v.GetArray(); err != nil {
		petsc.Fatal(err)
	}
	petsc.SyncPrintf("%d rank has local size %d \n", rank, len(v.Arr))
	petsc.SyncFlush()
	fmt.Println(rank, v.Arr[0:2])
	if err := v.RestoreArray(); err != nil {
		petsc.Fatal(err)
	}

	sum, _ := v.Sum()
	petsc.Printf("Sum = %f\n", sum)
	max, _, _ := v.Max()
	petsc.Printf("Max = %f\n", max)
	min, _, _ := v.Min()
	petsc.Printf("Max = %f\n", min)
	v.Scale(0.3)
	sum, _ = v.Sum()
	petsc.Printf("Sum = %f\n", sum)

	err = v.Destroy()
	if err != nil {
		petsc.Fatal(err)
	}

}
Example #10
0
func main() {
	// PETSc initialization
	if err := petsc.Initialize(); err != nil {
		petsc.Fatal(err)
	}
	defer func() {
		if err := petsc.Finalize(); err != nil {
			petsc.Fatal(err)
		}
	}()
	rank, _ := petsc.RankSize()

	v, err := structvec.NewStructVec(pstruct{}, petsc.DECIDE, 10)
	if err != nil {
		petsc.Fatal(err)
	}
	defer v.Destroy()
	petsc.Printf("Type of v : %s\n", v.Type())
	petsc.Printf("Size of v : %d\n", v.BlockSize())
	petsc.SyncPrintf("Local size = %d\n", v.Nlocal)
	petsc.SyncFlush()
	petsc.Printf("Global size = %d\n", v.Ntotal)

	// local particle data
	lpp, ok := v.GetArray().([]pstruct)
	if !ok {
		petsc.Fatal(err)
	}
	for i := range lpp {
		lpp[i].FillRandom()
	}
	err = v.RestoreArray()
	if err != nil {
		petsc.Fatal(err)
	}

	// Print array
	lpp, ok = v.GetArray().([]pstruct)
	if !ok {
		petsc.Fatal(err)
	}
	for i := range lpp {
		petsc.SyncPrintf("%s\n", lpp[i])
	}
	petsc.SyncFlush()
	err = v.RestoreArray()
	if err != nil {
		petsc.Fatal(err)
	}

	petsc.Printf("----------------\n")

	// Fiddle with array
	if rank == 0 {
		lpp = make([]pstruct, 2)
		ix := []int64{3, 7}
		err = v.SetValues(ix, lpp)
		if err != nil {
			petsc.Fatal(err)
		}
	}
	v.AssemblyBegin()
	v.AssemblyEnd()

	// Print array
	lpp, ok = v.GetArray().([]pstruct)
	if !ok {
		petsc.Fatal(err)
	}
	for i := range lpp {
		petsc.SyncPrintf("%s\n", lpp[i])
	}
	petsc.SyncFlush()
	err = v.RestoreArray()
	if err != nil {
		petsc.Fatal(err)
	}

}
Example #11
0
// Scatter takes the particles and reshuffles them across MPI ranks according to localid and
//
// localid is the local index of the particle, while mpirank is the destination rank for the particle.
// Note that this is completely general, and a particle may be shunted to many ranks (useful for ghosts).
// Also, note that localndx and mpirank will be modified by this routine.
//
func (s *StructVec) Scatter(localndx, mpirank []int64) {

	// Get the rank and size
	rank, size := petsc.RankSize()

	// Work out the final number of particles
	var npart_local, npart_final int64
	npart_local = int64(len(mpirank))
	mpi.AllReduceInt64(petsc.WORLD, &npart_local, &npart_final, 1, mpi.SUM)
	//petsc.Printf("%d total particles expected after scatter\n", npart_final)

	// Allocate arrays
	// narr[rank] are the number of particles headed for rank from the current rank (hereafter crank)
	// narr1[crank*size + rank] collects all the narr's, allowing every processor to know what index it needs to send objects to.
	// icount keeps track of indices that objects need to go to.
	// icount_check is used for assertion tests, to make sure nothing bad happened.
	narr := make([]int64, size)
	icount := make([]int64, size)
	narr1 := make([]int64, size*size)
	icount_check := make([]int64, size)

	// Loop over mpirank, incrementing narr
	for _, irank := range mpirank {
		narr[irank] += 1
	}
	mpi.AllGatherInt64(petsc.WORLD, narr, narr1)
	//petsc.Printf("%v\n", narr1)

	// Reset narr, icount
	for i := range narr {
		narr[i] = 0
		icount[i] = 0
	}

	for i := 0; i < size; i++ {
		// narr now holds the total number of local particles
		for j := 0; j < size; j++ {
			narr[i] += narr1[i+j*size]
		}
		// icount now holds the number of particles from ranks before my rank, on rank i
		for j := 0; j < rank; j++ {
			icount[i] += narr1[i+j*size]
		}
	}

	// Now switch icount to global indices
	// icount_check is the expected final global index
	rtot := int64(0)
	for i := 0; i < size; i++ {
		icount[i] += rtot
		rtot += narr[i]
		icount_check[i] = icount[i] + narr1[i+rank*size]
	}
	// Assertion check
	if rtot != npart_final {
		petsc.Fatal(errors.New("ASSERTION FAILURE : rtot != npart_final"))
	}

	// Now we start updating localndx and mpirank
	lo, _, _ := s.OwnRange()
	irank := int64(0)
	for i := range mpirank {
		localndx[i] += lo
		irank = mpirank[i]
		mpirank[i] = icount[irank]
		icount[irank]++
	}

	// Assertion check
	for i := range icount {
		if icount[i] != icount_check[i] {
			petsc.Fatal(errors.New("ASSERTION FAILURE : icount != icount_check"))
		}
	}

	// Create destination
	vecnew, err := petsc.NewVecBlocked(narr[rank]*s.bs, petsc.DETERMINE, s.bs)
	if err != nil {
		petsc.Fatal(err)
	}
	// Create index sets
	isin, err := petsc.NewBlockedIS(s.bs, npart_local, localndx)
	if err != nil {
		petsc.Fatal(err)
	}
	isout, err := petsc.NewBlockedIS(s.bs, npart_local, mpirank)
	if err != nil {
		petsc.Fatal(err)
	}
	defer isin.Destroy()
	defer isout.Destroy()

	// Create scatter context
	ctx, err := petsc.NewScatter(s.v, vecnew, isin, isout)
	defer ctx.Destroy()
	ctx.Begin(s.v, vecnew, false, true)
	ctx.End(s.v, vecnew, false, true)

	// Clean up
	s.v.Destroy()
	s.v = vecnew
	s.Nlocal = narr[rank]
	s.Ntotal = npart_final

}