func Cmp(feed1, feed2 MDQ) {
ents1, _ := feed1.getEntityList()
ents2, _ := feed2.getEntityList()
// len1 := len(ents1)
len2 := len(ents2)
seen := 0
for id, _ := range ents1 {
x1, _ := feed1.MDQ(id)
x2, _ := feed2.MDQ(id)
str1 := x1.Pp()
str2 := x2.Pp()
hash1 := hex.EncodeToString(gosaml.Hash(crypto.SHA1, str1))
hash2 := hex.EncodeToString(gosaml.Hash(crypto.SHA1, str2))
if hash1 == hash2 {
fmt.Println("OK ", id)
} else {
fmt.Println("NOT", id)
diffs := difflib.Diff(strings.Split(str1, "\n"), strings.Split(str2, "\n"))
for _, diff := range diffs {
if diff.Delta != difflib.Common {
fmt.Println(diff)
}
}
fmt.Println()
}
seen++
}
fmt.Println("Number of entityies: ", seen, len2)
}
func (mdq *MDQ) dbget(key string, cache bool) (xp *gosaml.Xp, err error) {
if mdq.stmt == nil {
mdq.stmt, err = mdq.db.Prepare(`select e.md from entity e, lookup l, validuntil v
where l.hash = $1 and l.entity_id_fk = e.id and v.validuntil >= $2`)
if err != nil {
return
}
}
k := key
const prefix = "{sha1}"
if strings.HasPrefix(key, prefix) {
key = key[6:]
} else {
key = hex.EncodeToString(gosaml.Hash(crypto.SHA1, key))
}
mdq.Lock.Lock()
defer mdq.Lock.Unlock()
cachedxp := mdq.Cache[key]
if cachedxp != nil && cachedxp.Valid(cacheduration) {
xp = cachedxp.Xp.CpXp()
return
}
var xml []byte
err = mdq.stmt.QueryRow(key, time.Now().Unix()).Scan(&xml)
if err != nil {
//log.Println("query", mdq.path, k, key, err, string(xml))
err = fmt.Errorf("Metadata not found for entity: %s", k)
//debug.PrintStack()
return
}
xp = gosaml.NewXp(xml)
if cache {
mdxp := new(MdXp)
mdxp.Xp = xp
mdxp.created = time.Now()
mdq.Cache[key] = mdxp
}
return
}
func WayfAttributeHandler(idp_md, hub_md, sp_md, response *gosaml.Xp) (err error) {
sourceAttributes := response.Query(nil, `/samlp:Response/saml:Assertion/saml:AttributeStatement`)[0]
idp := response.Query1(nil, "/samlp:Response/saml:Issuer")
attCS := hub_md.Query(nil, "./md:SPSSODescriptor/md:AttributeConsumingService")[0]
// First check for mandatory and multiplicity
requestedAttributes := hub_md.Query(attCS, `md:RequestedAttribute[not(@computed)]`) // [@isRequired='true' or @isRequired='1']`)
for _, requestedAttribute := range requestedAttributes {
name := requestedAttribute.GetAttr("Name")
friendlyName := requestedAttribute.GetAttr("FriendlyName")
//nameFormat := requestedAttribute.GetAttr("NameFormat")
mandatory := hub_md.QueryBool(requestedAttribute, "@mandatory")
//must := hub_md.QueryBool(requestedAttribute, "@must")
singular := hub_md.QueryBool(requestedAttribute, "@singular")
// accept attributes in both uri and basic format
attributes := response.Query(sourceAttributes, `saml:Attribute[@Name="`+name+`" or @Name="`+friendlyName+`"]`)
if len(attributes) == 0 && mandatory {
err = fmt.Errorf("mandatory: %s", friendlyName)
return
}
for _, attribute := range attributes {
valueNodes := response.Query(attribute, `saml:AttributeValue`)
if len(valueNodes) > 1 && singular {
err = fmt.Errorf("multiple values for singular attribute: %s", name)
return
}
if len(valueNodes) != 1 && mandatory {
err = fmt.Errorf("mandatory: %s", friendlyName)
return
}
attribute.SetAttr("Name", name)
attribute.SetAttr("FriendlyName", friendlyName)
attribute.SetAttr("NameFormat", uri)
}
}
// check that the security domain of eppn is one of the domains in the shib:scope list
// we just check that everything after the (leftmost|rightmost) @ is in the scope list and save the value for later
eppn := response.Query1(sourceAttributes, "saml:Attribute[@Name='urn:oid:1.3.6.1.4.1.5923.1.1.1.6']/saml:AttributeValue")
eppnregexp := regexp.MustCompile(`^[^\@]+\@([a-zA-Z0-9\.-]+)$`)
matches := eppnregexp.FindStringSubmatch(eppn)
if len(matches) != 2 {
err = fmt.Errorf("eppn does not seem to be an eppn: %s", eppn)
return
}
securitydomain := matches[1]
scope := idp_md.Query(nil, "//shibmd:Scope[.='"+securitydomain+"']")
if len(scope) == 0 {
err = fmt.Errorf("security domain '%s' for eppn does not match any scopes", securitydomain)
}
val := idp_md.Query1(nil, "./md:Extensions/wayf:wayf/wayf:wayf_schacHomeOrganizationType")
gosaml.CpAndSet(sourceAttributes, response, hub_md, attCS, "schacHomeOrganizationType", val)
val = idp_md.Query1(nil, "./md:Extensions/wayf:wayf/wayf:wayf_schacHomeOrganization")
gosaml.CpAndSet(sourceAttributes, response, hub_md, attCS, "schacHomeOrganization", val)
if response.Query1(sourceAttributes, `saml:Attribute[@FriendlyName="displayName"]/saml:AttributeValue`) == "" {
if cn := response.Query1(sourceAttributes, `saml:Attribute[@FriendlyName="cn"]/saml:AttributeValue`); cn != "" {
gosaml.CpAndSet(sourceAttributes, response, hub_md, attCS, "displayName", cn)
}
}
salt := "6xfkhc7juin4vlbetmmc0eyxumelnoku"
sp := sp_md.Query1(nil, "@entityID")
uidhashbase := "uidhashbase" + salt
uidhashbase += strconv.Itoa(len(idp)) + ":" + idp
uidhashbase += strconv.Itoa(len(sp)) + ":" + sp
uidhashbase += strconv.Itoa(len(eppn)) + ":" + eppn
uidhashbase += salt
eptid := "WAYF-DK-" + hex.EncodeToString(gosaml.Hash(crypto.SHA1, uidhashbase))
gosaml.CpAndSet(sourceAttributes, response, hub_md, attCS, "eduPersonTargetedID", eptid)
dkcprpreg := regexp.MustCompile(`^urn:mace:terena.org:schac:personalUniqueID:dk:CPR:(\d\d)(\d\d)(\d\d)(\d)\d\d\d$`)
for _, cprelement := range response.Query(sourceAttributes, `saml:Attribute[@FriendlyName="schacPersonalUniqueID"]`) {
// schacPersonalUniqueID is multi - use the first DK cpr found
cpr := strings.TrimSpace(response.NodeGetContent(cprelement))
if matches := dkcprpreg.FindStringSubmatch(cpr); len(matches) > 0 {
cpryear, _ := strconv.Atoi(matches[3])
c7, _ := strconv.Atoi(matches[4])
year := strconv.Itoa(yearfromyearandcifferseven(cpryear, c7))
gosaml.CpAndSet(sourceAttributes, response, hub_md, attCS, "schacDateOfBirth", year+matches[2]+matches[1])
gosaml.CpAndSet(sourceAttributes, response, hub_md, attCS, "schacYearOfBirth", year)
break
}
}
subsecuritydomain := "." + securitydomain
epsas := make(map[string]bool)
for _, epsa := range response.QueryMulti(sourceAttributes, `saml:Attribute[@FriendlyName="eduPersonScopedAffiliation"]/saml:AttributeValue`) {
epsa = strings.TrimSpace(epsa)
epsaparts := strings.SplitN(epsa, "@", 2)
if len(epsaparts) != 2 {
fmt.Errorf("eduPersonScopedAffiliation: %s does not end with a domain", epsa)
return
}
if !strings.HasSuffix(epsaparts[1], subsecuritydomain) && epsaparts[1] != securitydomain {
fmt.Printf("eduPersonScopedAffiliation: %s has not '%s' as a domain suffix", epsa, securitydomain)
return
}
epsas[epsa] = true
}
// primaryaffiliation => affiliation
epaAdd := []string{}
eppa := response.Query1(sourceAttributes, `saml:Attribute[@FriendlyName="eduPersonPrimaryAffiliation"]`)
eppa = strings.TrimSpace(eppa)
epas := response.QueryMulti(sourceAttributes, `saml:Attribute[@FriendlyName="eduPersonAffiliation"]`)
epaset := make(map[string]bool)
for _, epa := range epas {
epaset[strings.TrimSpace(epa)] = true
}
if !epaset[eppa] {
epaAdd = append(epaAdd, eppa)
epaset[eppa] = true
}
// 'student', 'faculty', 'staff', 'employee' => member
if epaset["student"] || epaset["faculty"] || epaset["staff"] || epaset["employee"] {
epaAdd = append(epaAdd, "member")
epaset["member"] = true
}
d := sourceAttributes.AddChild(hub_md.CopyNode(hub_md.Query(attCS, `md:RequestedAttribute[@FriendlyName="eduPersonAffiliation"]`)[0], 2))
for i, epa := range epaAdd {
response.QueryDashP(d, `saml:AttributeValue[`+strconv.Itoa(i+1)+`]`, epa, nil)
}
d = sourceAttributes.AddChild(hub_md.CopyNode(hub_md.Query(attCS, `md:RequestedAttribute[@FriendlyName="eduPersonScopedAffiliation"]`)[0], 2))
i := 1
for epa, _ := range epaset {
if epsas[epa] {
continue
}
response.QueryDashP(d, `saml:AttributeValue[`+strconv.Itoa(i)+`]`, epa+"@"+securitydomain, nil)
i += 1
}
return
// legal affiliations 'student', 'faculty', 'staff', 'affiliate', 'alum', 'employee', 'library-walk-in', 'member'
// affiliations => scopedaffiliations
}
func (mdq *MDQ) Update() (err error) {
start := time.Now()
log.Println("lMDQ updating", mdq.Url, mdq.Path)
_, err = mdq.db.Exec(lMDQSchema)
if err != nil {
return
}
recs, err := mdq.getEntityList()
if err != nil {
return err
}
var md []byte
if md, err = get(mdq.Url); err != nil {
return
}
dom := gosaml.NewXp(md)
if _, err := dom.SchemaValidate(mdq.MetadataSchemaPath); err != nil {
log.Println("feed", "SchemaError")
}
certificate := dom.Query(nil, "/md:EntitiesDescriptor/ds:Signature/ds:KeyInfo/ds:X509Data/ds:X509Certificate")
if len(certificate) != 1 {
err = errors.New("Metadata not signed")
return
}
keyname, key, err := gosaml.PublicKeyInfo(dom.NodeGetContent(certificate[0]))
if err != nil {
return
}
ok := dom.VerifySignature(nil, key)
if ok != nil || keyname != mdq.Hash {
return fmt.Errorf("Signature check failed. Signature %s, %s = %s", ok, keyname, mdq.Hash)
}
tx, err := mdq.db.Begin()
if err != nil {
return
}
defer func() {
if err != nil {
tx.Rollback()
return
}
err = tx.Commit()
}()
entityInsertStmt, err := tx.Prepare("insert into entity (entityid, md, hash) values ($1, $2, $3)")
if err != nil {
return
}
defer entityInsertStmt.Close()
lookupInsertStmt, err := tx.Prepare("insert or ignore into lookup (hash, entity_id_fk) values (?, ?)")
if err != nil {
return err
}
defer lookupInsertStmt.Close()
entityDeleteStmt, err := tx.Prepare("delete from entity where id = $1")
if err != nil {
return err
}
defer entityDeleteStmt.Close()
vu, err := time.Parse(time.RFC3339Nano, dom.Query1(nil, "@validUntil"))
if err != nil {
return err
}
validUntil := vu.Unix()
var new, updated, nochange, deleted int
seen := map[string]bool{}
entities := dom.Query(nil, "./md:EntityDescriptor")
for _, entity := range entities {
entityID := dom.Query1(entity, "@entityID")
if seen[entityID] {
log.Printf("lMDQ duplicate entityID: %s", entityID)
continue
}
seen[entityID] = true
md := gosaml.NewXpFromNode(entity).X2s()
rec := recs[entityID]
id := rec.id
hash := hex.EncodeToString(gosaml.Hash(crypto.SHA1, md))
oldhash := rec.hash
if rec.hash == hash { // no changes
delete(recs, entityID) // remove so it won't be deleted
nochange++
continue
} else if oldhash != "" { // update is delete + insert - then the cascading delete will also delete the potential stale lookup entries
_, err = entityDeleteStmt.Exec(rec.id)
if err != nil {
return
}
updated++
log.Printf("lMDQ updated entityID: %s", entityID)
delete(recs, entityID) // updated - remove so it won't be deleted
} else {
new++
if !mdq.Silent {
log.Printf("lMDQ new entityID: %s", entityID)
}
}
var res sql.Result
res, err = entityInsertStmt.Exec(entityID, md, hash)
if err != nil {
return err
}
id, _ = res.LastInsertId()
_, err = lookupInsertStmt.Exec(hex.EncodeToString(gosaml.Hash(crypto.SHA1, entityID)), id)
if err != nil {
return
}
for _, target := range indextargets {
locations := dom.Query(entity, target)
for i, location := range locations {
if !mdq.Silent {
log.Println(i, dom.NodeGetContent(location))
}
_, err = lookupInsertStmt.Exec(hex.EncodeToString(gosaml.Hash(crypto.SHA1, dom.NodeGetContent(location))), id)
if err != nil {
return
}
}
}
}
for entid, ent := range recs { // delete entities no longer in feed
_, err = entityDeleteStmt.Exec(ent.id)
if err != nil {
return
}
deleted++
log.Printf("lMDQ deleted entityID: %s", entid)
}
_, err = tx.Exec("update validuntil set validuntil = $1 where id = 1", validUntil)
if err != nil {
return
}
log.Printf("lMDQ finished %d new, %d updated, %d unchanged, %d deleted validUntil: %s duration: %.1f",
new, updated, nochange, deleted, time.Unix(validUntil, 0).Format(time.RFC3339), time.Since(start).Seconds())
return
}