Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions include/numerics/numeric_vector.h
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,24 @@ class NumericVector : public ReferenceCountedObject<NumericVector<T>>,

/**
* \returns The type (SERIAL, PARALLEL, GHOSTED) of the vector.
*
* This is metadata reflecting what type of vector was requested,
* which should reflect what level of replication and ghosting is
* necessary in parallel.
*/
ParallelType type() const { return _type; }

/**
* \returns Whether the vector is effectively serial, i.e. whether
* all vector data is accessible on every processor.
*
* This is true for explicitly SERIAL vectors, but also for vectors
* on serial (1-processor) communicators.
*/

bool is_effectively_serial() const
{ return (this->n_processors()==1) || (_type == SERIAL); }

/**
* \returns The type (SERIAL, PARALLEL, GHOSTED) of the vector.
*
Expand Down
23 changes: 12 additions & 11 deletions include/numerics/petsc_vector.h
Original file line number Diff line number Diff line change
Expand Up @@ -731,11 +731,7 @@ void PetscVector<T>::init (const numeric_index_type n,
if (this->initialized())
this->clear();

if (this->comm().size() == 1)
// This can help with some branching decisions and is also consistent with what PETSc does... a
// single rank Vec is going to be a sequential vector
this->_type = SERIAL;
else if (ptype == AUTOMATIC)
if (ptype == AUTOMATIC)
{
if (n == n_local)
this->_type = SERIAL;
Expand All @@ -745,17 +741,19 @@ void PetscVector<T>::init (const numeric_index_type n,
else
this->_type = ptype;

// We should have been given consistent settings
libmesh_assert ((this->_type==SERIAL && n==n_local) ||
(this->n_processors()==1 && n==n_local) ||
this->_type==PARALLEL);

// create a sequential vector if on only 1 processor
if (this->_type == SERIAL)
// create a sequential vector if on only 1 processor, or if asked
if (this->_type == SERIAL || (this->n_processors() == 1))
{
LibmeshPetscCallA(PETSC_COMM_SELF, VecCreate(PETSC_COMM_SELF, &_vec));
LibmeshPetscCallA(PETSC_COMM_SELF, VecSetSizes(_vec, petsc_n, petsc_n));
LibmeshPetscCallA(PETSC_COMM_SELF, VecSetFromOptions (_vec));
}
// otherwise create an MPI-enabled vector
// or create an MPI-enabled PARALLEL vector w/o ghosting if asked
else if (this->_type == PARALLEL)
{
#ifdef LIBMESH_HAVE_MPI
Expand All @@ -771,8 +769,11 @@ void PetscVector<T>::init (const numeric_index_type n,
#endif
LibmeshPetscCall(VecSetFromOptions(_vec));
}
// or yell because we don't know what to do
else
libmesh_error_msg("Unsupported type " << Utility::enum_to_string(this->_type));
libmesh_error_msg("Unsupported type " <<
Utility::enum_to_string(this->_type) <<
" for parallel init with no ghost indices supplied");

this->_is_initialized = true;
this->_is_closed = true;
Expand Down Expand Up @@ -801,14 +802,14 @@ void PetscVector<T>::init (const numeric_index_type n,
const numeric_index_type n_local,
const std::vector<numeric_index_type> & ghost,
const bool fast,
const ParallelType libmesh_dbg_var(ptype))
const ParallelType ptype)
{
parallel_object_only();

if (this->comm().size() == 1)
{
libmesh_assert(ghost.empty());
this->init(n, n_local, fast, SERIAL);
this->init(n, n_local, fast, ptype);
return;
}

Expand Down
15 changes: 10 additions & 5 deletions src/numerics/petsc_vector.C
Original file line number Diff line number Diff line change
Expand Up @@ -533,9 +533,9 @@ PetscVector<T>::operator = (const PetscVector<T> & v)
libmesh_assert (v.closed());

AssignmentType assign_type = Error;
if (this->type() == SERIAL && v.type() != SERIAL)
if (this->is_effectively_serial() && !v.is_effectively_serial())
assign_type = ParallelToSerial;
else if (this->type() != SERIAL && v.type() == SERIAL)
else if (!this->is_effectively_serial() && v.is_effectively_serial())
assign_type = SerialToParallel;
else if (this->local_size() == v.local_size())
assign_type = SameToSame;
Expand Down Expand Up @@ -1056,7 +1056,7 @@ void PetscVector<T>::create_subvector(NumericVector<T> & subvector,
parallel_object_only();

libmesh_error_msg_if(
subvector.type() == GHOSTED,
subvector.type() == GHOSTED && !subvector.is_effectively_serial(),
"We do not support scattering parallel information to ghosts for subvectors");

this->_restore_array();
Expand All @@ -1071,11 +1071,16 @@ void PetscVector<T>::create_subvector(NumericVector<T> & subvector,
// If not, we use the appropriate PETSc routines to initialize it.
if (!petsc_subvector->initialized())
{
if (this->type() == SERIAL)
if (this->is_effectively_serial())
{
libmesh_assert(this->comm().verify(rows.size()));
LibmeshPetscCall(VecCreateSeq(this->comm().get(), rows.size(), &(petsc_subvector->_vec)));
petsc_subvector->_type = SERIAL;

// You'd think we would create GHOSTED from GHOSTED, but we
// never have, and now there'd be downstream compatibility
// issues if we started even marking an effectively-serial
// subvector as GHOSTED.
petsc_subvector->_type = (this->type() == SERIAL) ? SERIAL : PARALLEL;
}
else
{
Expand Down